Skip to content

Commit

Permalink
Add SAS Token Authentication Support to Azure Repo Plugin (elastic#42982
Browse files Browse the repository at this point in the history 
) (elastic#43618)

* Added setting for SAS token
* Added support for the token in tests
* Relates elastic#42117
  • Loading branch information
@original-brownbear
original-brownbear committed Jul 1, 2019
1 parent 83eef2c commit 3fb343c
Show file tree
Hide file tree
Showing 7 changed files with 62 additions and 38 deletions.
10 changes: 7 additions & 3 deletions docs/plugins/repository-azure.asciidoc
View file
Expand Up @@ -19,7 +19,11 @@ bin/elasticsearch-keystore add azure.client.default.account
bin/elasticsearch-keystore add azure.client.default.key
----------------------------------------------------------------

Where `account` is the azure account name and `key` the azure secret key.
Where `account` is the azure account name and `key` the azure secret key. Instead of an azure secret key under `key`, you can alternatively
define a shared access signatures (SAS) token under `sas_token` to use for authentication instead. When using an SAS token instead of an
account key, the SAS token must have read (r), write (w), list (l), and delete (d) permissions for the repository base path and
all its contents. These permissions need to be granted for the blob service (b) and apply to resource types service (s), container (c), and
object (o).
These settings are used by the repository's internal azure client.

Note that you can also define more than one account:
Expand All @@ -29,14 +33,14 @@ Note that you can also define more than one account:
bin/elasticsearch-keystore add azure.client.default.account
bin/elasticsearch-keystore add azure.client.default.key
bin/elasticsearch-keystore add azure.client.secondary.account
bin/elasticsearch-keystore add azure.client.secondary.key
bin/elasticsearch-keystore add azure.client.secondary.sas_token
----------------------------------------------------------------

`default` is the default account name which will be used by a repository,
unless you set an explicit one in the
<<repository-azure-repository-settings, repository settings>>.

Both `account` and `key` storage settings are
The `account`, `key`, and `sas_token` storage settings are
{ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
reload the settings, the internal azure clients, which are used to transfer the
snapshot, will utilize the latest settings from the keystore.
Expand Down
4 changes: 3 additions & 1 deletion plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
View file
Expand Up @@ -33,12 +33,14 @@ String azureAccount = System.getenv("azure_storage_account")
String azureKey = System.getenv("azure_storage_key")
String azureContainer = System.getenv("azure_storage_container")
String azureBasePath = System.getenv("azure_storage_base_path")
String azureSasToken = System.getenv("azure_storage_sas_token")

if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) {
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
azureAccount = 'azure_integration_test_account'
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
azureContainer = 'container_test'
azureBasePath = 'integration_test'
azureSasToken = ''
useFixture = true
}

Expand Down
1 change: 1 addition & 0 deletions ...itory-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
View file
Expand Up @@ -59,6 +59,7 @@ public List<Setting<?>> getSettings() {
AzureStorageSettings.Storage.STORAGE_ACCOUNTS,
AzureStorageSettings.ACCOUNT_SETTING,
AzureStorageSettings.KEY_SETTING,
AzureStorageSettings.SAS_TOKEN_SETTING,
AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
AzureStorageSettings.TIMEOUT_SETTING,
AzureStorageSettings.MAX_RETRIES_SETTING,
Expand Down
4 changes: 2 additions & 2 deletions ...ository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
View file
Expand Up @@ -112,8 +112,8 @@ protected CloudBlobClient buildClient(AzureStorageSettings azureStorageSettings)
return client;
}

protected CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
final String connectionString = azureStorageSettings.buildConnectionString();
private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
final String connectionString = azureStorageSettings.getConnectString();
return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
}

Expand Down
66 changes: 40 additions & 26 deletions ...sitory-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
View file
Expand Up @@ -21,6 +21,7 @@

import com.microsoft.azure.storage.LocationMode;
import com.microsoft.azure.storage.RetryPolicy;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
Expand Down Expand Up @@ -57,6 +58,10 @@ public final class AzureStorageSettings {
public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key",
key -> SecureSetting.secureString(key, null));

/** Azure SAS token */
public static final AffixSetting<SecureString> SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token",
key -> SecureSetting.secureString(key, null));

/** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
public static final Setting<Integer> MAX_RETRIES_SETTING =
Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
Expand Down Expand Up @@ -118,7 +123,7 @@ public interface Storage {
@Deprecated
private final String name;
private final String account;
private final String key;
private final String connectString;
private final String endpointSuffix;
private final TimeValue timeout;
@Deprecated
Expand All @@ -128,11 +133,11 @@ public interface Storage {
private final LocationMode locationMode;

// copy-constructor
private AzureStorageSettings(String name, String account, String key, String endpointSuffix, TimeValue timeout, boolean activeByDefault,
int maxRetries, Proxy proxy, LocationMode locationMode) {
private AzureStorageSettings(String name, String account, String connectString, String endpointSuffix, TimeValue timeout,
boolean activeByDefault, int maxRetries, Proxy proxy, LocationMode locationMode) {
this.name = name;
this.account = account;
this.key = key;
this.connectString = connectString;
this.endpointSuffix = endpointSuffix;
this.timeout = timeout;
this.activeByDefault = activeByDefault;
Expand All @@ -145,7 +150,7 @@ private AzureStorageSettings(String name, String account, String key, String end
public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault, int maxRetries) {
this.name = name;
this.account = account;
this.key = key;
this.connectString = buildConnectString(account, key, null, null);
this.endpointSuffix = null;
this.timeout = timeout;
this.activeByDefault = activeByDefault;
Expand All @@ -154,11 +159,11 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t
this.locationMode = LocationMode.PRIMARY_ONLY;
}

AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries,
Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
this.name = null;
AzureStorageSettings(String name, String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries,
Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
this.name = name;
this.account = account;
this.key = key;
this.connectString = buildConnectString(account, key, sasToken, endpointSuffix);
this.endpointSuffix = endpointSuffix;
this.timeout = timeout;
this.activeByDefault = false;
Expand Down Expand Up @@ -189,10 +194,6 @@ public String getName() {
return name;
}

public String getKey() {
return key;
}

public String getAccount() {
return account;
}
Expand All @@ -218,13 +219,26 @@ public Proxy getProxy() {
return proxy;
}

public String buildConnectionString() {
public String getConnectString() {
return connectString;
}

private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) {
final boolean hasSasToken = Strings.hasText(sasToken);
final boolean hasKey = Strings.hasText(key);
if (hasSasToken == false && hasKey == false) {
throw new SettingsException("Neither a secret key nor a shared access token was set.");
}
if (hasSasToken && hasKey) {
throw new SettingsException("Both a secret as well as a shared access token were set.");
}
final StringBuilder connectionStringBuilder = new StringBuilder();
connectionStringBuilder.append("DefaultEndpointsProtocol=https")
.append(";AccountName=")
.append(account)
.append(";AccountKey=")
.append(key);
connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account);
if (hasKey) {
connectionStringBuilder.append(";AccountKey=").append(key);
} else {
connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken);
}
if (Strings.hasText(endpointSuffix)) {
connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
}
Expand All @@ -239,7 +253,6 @@ public LocationMode getLocationMode() {
public String toString() {
final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
sb.append("account='").append(account).append('\'');
sb.append(", key='").append(key).append('\'');
sb.append(", activeByDefault='").append(activeByDefault).append('\'');
sb.append(", timeout=").append(timeout);
sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
Expand Down Expand Up @@ -309,8 +322,9 @@ static Map<String, AzureStorageSettings> loadRegular(Settings settings) {
/** Parse settings for a single client. */
static AzureStorageSettings getClientSettings(Settings settings, String clientName) {
try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
return new AzureStorageSettings(account.toString(), key.toString(),
SecureString key = getConfigValue(settings, clientName, KEY_SETTING);
SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) {
return new AzureStorageSettings(null, account.toString(), key.toString(), sasToken.toString(),
getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
getValue(settings, clientName, TIMEOUT_SETTING),
getValue(settings, clientName, MAX_RETRIES_SETTING),
Expand Down Expand Up @@ -358,8 +372,8 @@ private static AzureStorageSettings getPrimary(List<AzureStorageSettings> settin
} else if (settings.size() == 1) {
// the only storage settings belong (implicitly) to the default primary storage
AzureStorageSettings storage = settings.get(0);
return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true,
storage.getMaxRetries());
return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.connectString, null, storage.getTimeout(),
true, storage.getMaxRetries(), null, LocationMode.PRIMARY_ONLY);
} else {
AzureStorageSettings primary = null;
for (AzureStorageSettings setting : settings) {
Expand Down Expand Up @@ -398,8 +412,8 @@ public static Map<String, AzureStorageSettings> overrideLocationMode(Map<String,
final MapBuilder<String, AzureStorageSettings> mapBuilder = new MapBuilder<>();
for (final Map.Entry<String, AzureStorageSettings> entry : clientsSettings.entrySet()) {
final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().name, entry.getValue().account,
entry.getValue().key, entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().activeByDefault,
entry.getValue().maxRetries, entry.getValue().proxy, locationMode);
entry.getValue().connectString, entry.getValue().endpointSuffix, entry.getValue().timeout,
entry.getValue().activeByDefault, entry.getValue().maxRetries, entry.getValue().proxy, locationMode);
mapBuilder.put(entry.getKey(), azureSettings);
}
return mapBuilder.immutableMap();
Expand Down
3 changes: 0 additions & 3 deletions ...ry-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java
View file
Expand Up @@ -49,11 +49,9 @@ public void testParseTwoSettingsExplicitDefault() {
Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.loadLegacy(settings);
assertThat(tuple.v1(), notNullValue());
assertThat(tuple.v1().getAccount(), is("myaccount1"));
assertThat(tuple.v1().getKey(), is("mykey1"));
assertThat(tuple.v2().keySet(), hasSize(1));
assertThat(tuple.v2().get("azure2"), notNullValue());
assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2"));
assertThat(tuple.v2().get("azure2").getKey(), is("mykey2"));
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"),
Expand All @@ -72,7 +70,6 @@ public void testParseUniqueSettings() {
Tuple<AzureStorageSettings, Map<String, AzureStorageSettings>> tuple = AzureStorageSettings.loadLegacy(settings);
assertThat(tuple.v1(), notNullValue());
assertThat(tuple.v1().getAccount(), is("myaccount1"));
assertThat(tuple.v1().getKey(), is("mykey1"));
assertThat(tuple.v2().keySet(), hasSize(0));
assertSettingDeprecationsAndWarnings(new Setting<?>[]{
getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"),
Expand Down
12 changes: 9 additions & 3 deletions ...ry-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
View file
Expand Up @@ -176,15 +176,21 @@ public void testReinitClientWrongSettings() throws IOException {
secureSettings2.setString("azure.client.azure1.account", "myaccount1");
// missing key
final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
final MockSecureSettings secureSettings3 = new MockSecureSettings();
secureSettings3.setString("azure.client.azure1.account", "myaccount3");
secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33"));
secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
final AzureStorageService azureStorageService = plugin.azureStoreService;
final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
plugin.reload(settings2);
final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
// existing client untouched
assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure1"));
assertThat(e.getMessage(), is("Invalid azure client settings with name [azure1]"));
}
}

Expand Down

0 comments on commit 3fb343c

Please sign in to comment.