Skip to content
Permalink
Browse files
Merge pull request #501 from jelmini/OAK-9704
OAK-9704 AzureBlobStoreBackend: empty string as null in boolean property
  • Loading branch information
smiroslav committed Feb 23, 2022
2 parents bfa326d + 0994fbb commit bd8a55f5a5ce0bdda23319e06bd0422c90bdeacb
Showing 5 changed files with 367 additions and 15 deletions.
@@ -174,6 +174,13 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.arakelian</groupId>
<artifactId>docker-junit-rule</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

</project>
@@ -164,12 +164,13 @@ public void init() throws DataStoreException {

try {
Utils.setProxyIfNeeded(properties);
containerName = (String) properties.get(AzureConstants.AZURE_BLOB_CONTAINER_NAME);
createBlobContainer = PropertiesUtil.toBoolean(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER), true);
containerName = properties.getProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME);
createBlobContainer = PropertiesUtil.toBoolean(
Strings.emptyToNull(properties.getProperty(AzureConstants.AZURE_CREATE_CONTAINER)), true);
connectionString = Utils.getConnectionStringFromProperties(properties);

concurrentRequestCount = PropertiesUtil.toInteger(
properties.get(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
properties.getProperty(AzureConstants.AZURE_BLOB_CONCURRENT_REQUESTS_PER_OPERATION),
DEFAULT_CONCURRENT_REQUEST_COUNT);
if (concurrentRequestCount < DEFAULT_CONCURRENT_REQUEST_COUNT) {
LOG.warn("Invalid setting [{}] for concurrentRequestsPerOperation (too low); resetting to {}",
@@ -184,12 +185,12 @@ public void init() throws DataStoreException {
}
LOG.info("Using concurrentRequestsPerOperation={}", concurrentRequestCount);

retryPolicy = Utils.getRetryPolicy((String)properties.get(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
retryPolicy = Utils.getRetryPolicy(properties.getProperty(AzureConstants.AZURE_BLOB_MAX_REQUEST_RETRY));
if (properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT) != null) {
requestTimeout = PropertiesUtil.toInteger(properties.getProperty(AzureConstants.AZURE_BLOB_REQUEST_TIMEOUT), RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT);
}
presignedDownloadURIVerifyExists =
PropertiesUtil.toBoolean(properties.get(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS), true);
presignedDownloadURIVerifyExists = PropertiesUtil.toBoolean(
Strings.emptyToNull(properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS)), true);

CloudBlobContainer azureContainer = getAzureContainer();

@@ -253,11 +254,11 @@ public InputStream read(DataIdentifier identifier) throws DataStoreException {
return is;
}
catch (StorageException e) {
LOG.info("Error reading blob. identifier=%s", key);
LOG.info("Error reading blob. identifier={}", key);
throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e);
}
catch (URISyntaxException e) {
LOG.debug("Error reading blob. identifier=%s", key);
LOG.debug("Error reading blob. identifier={}", key);
throw new DataStoreException(String.format("Cannot read blob. identifier=%s", key), e);
} finally {
if (contextClassLoader != null) {
@@ -141,7 +141,8 @@ public static String getConnectionStringFromProperties(Properties properties) {

return getConnectionString(
accountName,
accountKey);
accountKey,
blobEndpoint);
}

private static String getConnectionStringForSas(String sasUri, String blobEndpoint, String accountName) {
@@ -152,12 +153,15 @@ private static String getConnectionStringForSas(String sasUri, String blobEndpoi
}
}

public static String getConnectionString(final String accountName, final String accountKey) {
return String.format(
"DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s",
accountName,
accountKey
);
public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) {
StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https");
connString.append(";AccountName=").append(accountName);
connString.append(";AccountKey=").append(accountKey);

if (!Strings.isNullOrEmpty(blobEndpoint)) {
connString.append(";BlobEndpoint=").append(blobEndpoint);
}
return connString.toString();
}

/**
@@ -0,0 +1,239 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;

import com.google.common.collect.ImmutableSet;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
import java.io.IOException;
import java.net.URISyntaxException;
import java.time.Duration;
import java.time.Instant;
import java.util.Date;
import java.util.EnumSet;
import java.util.Properties;
import java.util.Set;
import java.util.stream.StreamSupport;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.ClassRule;
import org.junit.Test;

import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.ADD;
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.CREATE;
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.LIST;
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
import static com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
import static java.util.stream.Collectors.toSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

public class AzureBlobStoreBackendTest {
@ClassRule
public static AzuriteDockerRule azurite = new AzuriteDockerRule();

private static final String CONTAINER_NAME = "blobstore";
private static final EnumSet<SharedAccessBlobPermissions> READ_ONLY = EnumSet.of(READ, LIST);
private static final EnumSet<SharedAccessBlobPermissions> READ_WRITE = EnumSet.of(READ, LIST, CREATE, WRITE, ADD);
private static final ImmutableSet<String> BLOBS = ImmutableSet.of("blob1", "blob2");

private CloudBlobContainer container;

@After
public void tearDown() throws Exception {
if (container != null) {
container.deleteIfExists();
}
}

@Test
public void initWithSharedAccessSignature_readOnly() throws Exception {
CloudBlobContainer container = createBlobContainer();
String sasToken = container.generateSharedAccessSignature(policy(READ_ONLY), null);

AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));

azureBlobStoreBackend.init();

assertWriteAccessNotGranted(azureBlobStoreBackend);
assertReadAccessGranted(azureBlobStoreBackend, BLOBS);
}

@Test
public void initWithSharedAccessSignature_readWrite() throws Exception {
CloudBlobContainer container = createBlobContainer();
String sasToken = container.generateSharedAccessSignature(policy(READ_WRITE), null);

AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));

azureBlobStoreBackend.init();

assertWriteAccessGranted(azureBlobStoreBackend, "file");
assertReadAccessGranted(azureBlobStoreBackend, concat(BLOBS, "file"));
}

@Test
public void connectWithSharedAccessSignatureURL_expired() throws Exception {
CloudBlobContainer container = createBlobContainer();
SharedAccessBlobPolicy expiredPolicy = policy(READ_WRITE, yesterday());
String sasToken = container.generateSharedAccessSignature(expiredPolicy, null);

AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithSasToken(sasToken));

azureBlobStoreBackend.init();

assertWriteAccessNotGranted(azureBlobStoreBackend);
assertReadAccessNotGranted(azureBlobStoreBackend);
}

@Test
public void initWithAccessKey() throws Exception {
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithAccessKey());

azureBlobStoreBackend.init();

assertWriteAccessGranted(azureBlobStoreBackend, "file");
assertReadAccessGranted(azureBlobStoreBackend, ImmutableSet.of("file"));
}

@Test
public void initWithConnectionURL() throws Exception {
AzureBlobStoreBackend azureBlobStoreBackend = new AzureBlobStoreBackend();
azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());

azureBlobStoreBackend.init();

assertWriteAccessGranted(azureBlobStoreBackend, "file");
assertReadAccessGranted(azureBlobStoreBackend, ImmutableSet.of("file"));
}

private CloudBlobContainer createBlobContainer() throws Exception {
container = azurite.getContainer("blobstore");
for (String blob : BLOBS) {
container.getBlockBlobReference(blob + ".txt").uploadText(blob);
}
return container;
}

private static Properties getConfigurationWithSasToken(String sasToken) {
Properties properties = getBasicConfiguration();
properties.setProperty(AzureConstants.AZURE_SAS, sasToken);
properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "false");
return properties;
}

private static Properties getConfigurationWithAccessKey() {
Properties properties = getBasicConfiguration();
properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY, AzuriteDockerRule.ACCOUNT_KEY);
return properties;
}

@NotNull
private static Properties getConfigurationWithConnectionString() {
Properties properties = getBasicConfiguration();
properties.setProperty(AzureConstants.AZURE_CONNECTION_STRING, getConnectionString());
return properties;
}

@NotNull
private static Properties getBasicConfiguration() {
Properties properties = new Properties();
properties.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, CONTAINER_NAME);
properties.setProperty(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_NAME);
properties.setProperty(AzureConstants.AZURE_BLOB_ENDPOINT, azurite.getBlobEndpoint());
properties.setProperty(AzureConstants.AZURE_CREATE_CONTAINER, "");
return properties;
}

@NotNull
private static SharedAccessBlobPolicy policy(EnumSet<SharedAccessBlobPermissions> permissions, Instant expirationTime) {
SharedAccessBlobPolicy sharedAccessBlobPolicy = new SharedAccessBlobPolicy();
sharedAccessBlobPolicy.setPermissions(permissions);
sharedAccessBlobPolicy.setSharedAccessExpiryTime(Date.from(expirationTime));
return sharedAccessBlobPolicy;
}

@NotNull
private static SharedAccessBlobPolicy policy(EnumSet<SharedAccessBlobPermissions> permissions) {
return policy(permissions, Instant.now().plus(Duration.ofDays(7)));
}

private static void assertReadAccessGranted(AzureBlobStoreBackend backend, Set<String> expectedBlobs) throws Exception {
CloudBlobContainer container = backend.getAzureContainer();
Set<String> actualBlobNames = StreamSupport.stream(container.listBlobs().spliterator(), false)
.map(blob -> blob.getUri().getPath())
.map(path -> path.substring(path.lastIndexOf('/') + 1))
.collect(toSet());
Set<String> expectedBlobNames = expectedBlobs.stream().map(name -> name + ".txt").collect(toSet());

assertEquals(expectedBlobNames, actualBlobNames);

Set<String> actualBlobContent = actualBlobNames.stream()
.map(name -> {
try {
return container.getBlockBlobReference(name).downloadText();
} catch (StorageException | IOException | URISyntaxException e) {
throw new RuntimeException("Error while reading blob " + name, e);
}
})
.collect(toSet());
assertEquals(expectedBlobs, actualBlobContent);
}

private static void assertWriteAccessGranted(AzureBlobStoreBackend backend, String blob) throws Exception {
backend.getAzureContainer()
.getBlockBlobReference(blob + ".txt").uploadText(blob);
}

private static void assertWriteAccessNotGranted(AzureBlobStoreBackend backend) {
try {
assertWriteAccessGranted(backend, "test.txt");
fail("Write access should not be granted, but writing to the storage succeeded.");
} catch (Exception e) {
// successful
}
}

private static void assertReadAccessNotGranted(AzureBlobStoreBackend backend) {
try {
assertReadAccessGranted(backend, BLOBS);
fail("Read access should not be granted, but reading from the storage succeeded.");
} catch (Exception e) {
// successful
}
}

private static Instant yesterday() {
return Instant.now().minus(Duration.ofDays(1));
}

private static ImmutableSet<String> concat(ImmutableSet<String> set, String element) {
return ImmutableSet.<String>builder().addAll(set).add(element).build();
}

private static String getConnectionString() {
return Utils.getConnectionString(AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getBlobEndpoint());
}
}

0 comments on commit bd8a55f

Please sign in to comment.