Blob Storage and DICOM Blob Data Store

Microsoft.Health.Dicom.sln

@@ -12,10 +12,18 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Web"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{8C9A0050-5D22-4398-9F93-DDCD80B3BA51}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.Web.Tests.E2E", "test\Microsoft.Health.Dicom.Web.Tests.E2E\Microsoft.Health.Dicom.Web.Tests.E2E.csproj", "{1D0ECFDA-2AF2-4796-995D-A7C6E18C9CD1}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Web.Tests.E2E", "test\Microsoft.Health.Dicom.Web.Tests.E2E\Microsoft.Health.Dicom.Web.Tests.E2E.csproj", "{1D0ECFDA-2AF2-4796-995D-A7C6E18C9CD1}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{176641B3-297C-4E04-A83D-8F80F80485E8}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Blob", "src\Microsoft.Health.Blob\Microsoft.Health.Blob.csproj", "{7DBF0671-18D7-4EC2-A0B0-05AA97D73DDE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Dicom.Blob", "src\Microsoft.Health.Dicom.Blob\Microsoft.Health.Dicom.Blob.csproj", "{1FFFABFB-B30A-4AB8-8193-67016B1C5276}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Blob.UnitTests", "src\Microsoft.Health.Blob.UnitTests\Microsoft.Health.Blob.UnitTests.csproj", "{875E8062-FF09-42BB-8244-8C6145C95E5B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Dicom.Tests.Integration", "test\Microsoft.Health.Dicom.Tests.Integration\Microsoft.Health.Dicom.Tests.Integration.csproj", "{DFB41ECC-726C-4DBA-8AD3-17FB0A2546CA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -30,16 +38,36 @@ Global
 		{1D0ECFDA-2AF2-4796-995D-A7C6E18C9CD1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1D0ECFDA-2AF2-4796-995D-A7C6E18C9CD1}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1D0ECFDA-2AF2-4796-995D-A7C6E18C9CD1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7DBF0671-18D7-4EC2-A0B0-05AA97D73DDE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7DBF0671-18D7-4EC2-A0B0-05AA97D73DDE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7DBF0671-18D7-4EC2-A0B0-05AA97D73DDE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7DBF0671-18D7-4EC2-A0B0-05AA97D73DDE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1FFFABFB-B30A-4AB8-8193-67016B1C5276}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1FFFABFB-B30A-4AB8-8193-67016B1C5276}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1FFFABFB-B30A-4AB8-8193-67016B1C5276}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1FFFABFB-B30A-4AB8-8193-67016B1C5276}.Release|Any CPU.Build.0 = Release|Any CPU
+		{875E8062-FF09-42BB-8244-8C6145C95E5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{875E8062-FF09-42BB-8244-8C6145C95E5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{875E8062-FF09-42BB-8244-8C6145C95E5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{875E8062-FF09-42BB-8244-8C6145C95E5B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DFB41ECC-726C-4DBA-8AD3-17FB0A2546CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DFB41ECC-726C-4DBA-8AD3-17FB0A2546CA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DFB41ECC-726C-4DBA-8AD3-17FB0A2546CA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DFB41ECC-726C-4DBA-8AD3-17FB0A2546CA}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{BFB96311-9B1A-41C1-ABF1-4F6522660084} = {176641B3-297C-4E04-A83D-8F80F80485E8}
 		{1D0ECFDA-2AF2-4796-995D-A7C6E18C9CD1} = {8C9A0050-5D22-4398-9F93-DDCD80B3BA51}
+		{7DBF0671-18D7-4EC2-A0B0-05AA97D73DDE} = {176641B3-297C-4E04-A83D-8F80F80485E8}
+		{1FFFABFB-B30A-4AB8-8193-67016B1C5276} = {176641B3-297C-4E04-A83D-8F80F80485E8}
+		{875E8062-FF09-42BB-8244-8C6145C95E5B} = {176641B3-297C-4E04-A83D-8F80F80485E8}
+		{DFB41ECC-726C-4DBA-8AD3-17FB0A2546CA} = {8C9A0050-5D22-4398-9F93-DDCD80B3BA51}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
-		RESX_SortFileContentOnSave = True
 		SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8}
+		RESX_SortFileContentOnSave = True
 	EndGlobalSection
 EndGlobal

build/build.yml

@@ -1,5 +1,4 @@
 parameters:
-  # Default values
   packageArtifacts: true
 
 steps:
@@ -10,6 +9,26 @@ steps:
 - script: dotnet build --configuration $(buildConfiguration) --version-suffix $(build.buildNumber) /warnaserror
   displayName: 'dotnet build $(buildConfiguration)'
 
+# To run the tests we install NPM to run the 'azurite' emulator (https://github.com/Azure/Azurite).
+# This allows us to test the blob storage providers without an Azure instance.
+- task: NodeTool@0
+  displayName: 'Use Node 8.x'
+  inputs:
+    versionSpec: 8.x
+    checkLatest: true
+
+- script: npm install -g azurite@2.7.0
+  displayName: 'Install Azurite v2.7.0'
+
+# We start the Azurite as a separate process as the start call is blocking.
+- script: azurite -s &
+  condition: and(succeeded(), eq( variables['Agent.OS'], 'Linux' ))
+  displayName: 'Start Azurite Storage Emulator (Linux)'
+
+- script: start azurite -s
+  condition: and(succeeded(), eq( variables['Agent.OS'], 'Windows_NT' ))
+  displayName: 'Start Azurite Storage Emulator (Windows)'
+
 - task: DotNetCoreCLI@2
   displayName: 'dotnet test UnitTests'
   inputs:

src/Microsoft.Health.Blob.UnitTests/Features/Health/BlobHealthCheckTests.cs

@@ -0,0 +1,60 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Azure.Storage.Blob;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.Health.Blob.Configs;
+using Microsoft.Health.Blob.Features.Storage;
+using NSubstitute;
+using NSubstitute.ExceptionExtensions;
+using Xunit;
+
+namespace Microsoft.Health.Blob.UnitTests.Features.Health
+{
+    public class BlobHealthCheckTests
+    {
+        private readonly CloudBlobClient _client = Substitute.For<CloudBlobClient>(new Uri("https://www.microsoft.com/"), null);
+        private readonly IBlobClientTestProvider _testProvider = Substitute.For<IBlobClientTestProvider>();
+        private readonly BlobDataStoreConfiguration _configuration = new BlobDataStoreConfiguration { };
+        private readonly BlobContainerConfiguration _containerConfiguration = new BlobContainerConfiguration { ContainerName = "mycont" };
+
+        private readonly TestBlobHealthCheck _healthCheck;
+
+        public BlobHealthCheckTests()
+        {
+            IOptionsSnapshot<BlobContainerConfiguration> optionsSnapshot = Substitute.For<IOptionsSnapshot<BlobContainerConfiguration>>();
+            optionsSnapshot.Get(TestBlobHealthCheck.TestBlobHealthCheckName).Returns(_containerConfiguration);
+
+            _healthCheck = new TestBlobHealthCheck(
+                _client,
+                _configuration,
+                optionsSnapshot,
+                _testProvider,
+                NullLogger<TestBlobHealthCheck>.Instance);
+        }
+
+        [Fact]
+        public async Task GivenCosmosDbCanBeQueried_WhenHealthIsChecked_ThenHealthyStateShouldBeReturned()
+        {
+            HealthCheckResult result = await _healthCheck.CheckHealthAsync(new HealthCheckContext());
+
+            Assert.Equal(HealthStatus.Healthy, result.Status);
+        }
+
+        [Fact]
+        public async Task GivenCosmosDbCannotBeQueried_WhenHealthIsChecked_ThenUnhealthyStateShouldBeReturned()
+        {
+            _testProvider.PerformTestAsync(default, default, _containerConfiguration).ThrowsForAnyArgs<HttpRequestException>();
+            HealthCheckResult result = await _healthCheck.CheckHealthAsync(new HealthCheckContext());
+
+            Assert.Equal(HealthStatus.Unhealthy, result.Status);
+        }
+    }
+}

src/Microsoft.Health.Blob.UnitTests/Features/Health/TestBlobHealthCheck.cs

@@ -0,0 +1,35 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using Microsoft.Azure.Storage.Blob;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Microsoft.Health.Blob.Configs;
+using Microsoft.Health.Blob.Features.Health;
+using Microsoft.Health.Blob.Features.Storage;
+
+namespace Microsoft.Health.Blob.UnitTests.Features.Health
+{
+    internal class TestBlobHealthCheck : BlobHealthCheck
+    {
+        public const string TestBlobHealthCheckName = "TestBlobHealthCheck";
+
+        public TestBlobHealthCheck(
+            CloudBlobClient client,
+            BlobDataStoreConfiguration configuration,
+            IOptionsSnapshot<BlobContainerConfiguration> namedBlobContainerConfigurationAccessor,
+            IBlobClientTestProvider testProvider,
+            ILogger<TestBlobHealthCheck> logger)
+            : base(
+                  client,
+                  configuration,
+                  namedBlobContainerConfigurationAccessor,
+                  TestBlobHealthCheckName,
+                  testProvider,
+                  logger)
+        {
+        }
+    }
+}

src/Microsoft.Health.Blob.UnitTests/Features/Storage/BlobClientInitializerTests.cs

@@ -0,0 +1,71 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Azure.Storage.Blob;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Health.Blob.Configs;
+using Microsoft.Health.Blob.Features.Storage;
+using NSubstitute;
+using Xunit;
+
+namespace Microsoft.Health.Blob.UnitTests.Features.Storage
+{
+    public class BlobClientInitializerTests
+    {
+        private const string TestContainerName1 = "testcontainer1";
+        private const string TestContainerName2 = "testcontainer2";
+        private readonly IBlobClientInitializer _blobClientInitializer;
+        private readonly CloudBlobClient _blobClient;
+        private readonly IBlobContainerInitializer _containerInitializer1;
+        private readonly IBlobContainerInitializer _containerInitializer2;
+        private readonly List<IBlobContainerInitializer> _collectionInitializers;
+        private readonly CloudBlobContainer _cloudBlobContainer1;
+        private readonly CloudBlobContainer _cloudBlobContainer2;
+        private readonly BlobDataStoreConfiguration _blobDataStoreConfiguration = new BlobDataStoreConfiguration { };
+
+        public BlobClientInitializerTests()
+        {
+            _cloudBlobContainer1 = Substitute.For<CloudBlobContainer>(new Uri("https://www.microsoft.com/"));
+            _cloudBlobContainer2 = Substitute.For<CloudBlobContainer>(new Uri("https://www.microsoft.com/"));
+
+            IBlobClientTestProvider blobClientTestProvider = Substitute.For<IBlobClientTestProvider>();
+            _blobClient = Substitute.For<CloudBlobClient>(new Uri("https://www.microsoft.com/"), null);
+            _blobClient.GetContainerReference(TestContainerName1).Returns(_cloudBlobContainer1);
+            _blobClient.GetContainerReference(TestContainerName2).Returns(_cloudBlobContainer2);
+
+            _blobClientInitializer = new BlobClientInitializer(blobClientTestProvider, NullLogger<BlobClientInitializer>.Instance);
+            _containerInitializer1 = Substitute.For<BlobContainerInitializer>(TestContainerName1, NullLogger<BlobContainerInitializer>.Instance);
+            _containerInitializer2 = Substitute.For<BlobContainerInitializer>(TestContainerName2, NullLogger<BlobContainerInitializer>.Instance);
+            _collectionInitializers = new List<IBlobContainerInitializer> { _containerInitializer1, _containerInitializer2 };
+        }
+
+        [Fact]
+        public async void GivenMultipleCollections_WhenInitializing_ThenEachContainerInitializeMethodIsCalled()
+        {
+            await _blobClientInitializer.InitializeDataStoreAsync(_blobClient, _blobDataStoreConfiguration, _collectionInitializers);
+
+            await _containerInitializer1.Received(1).InitializeContainerAsync(_blobClient);
+            await _containerInitializer2.Received(1).InitializeContainerAsync(_blobClient);
+        }
+
+        [Fact]
+        public async void GivenAConfiguration_WhenInitializing_ThenCreateContainerIfNotExistsIsCalled()
+        {
+            await _blobClientInitializer.InitializeDataStoreAsync(_blobClient, _blobDataStoreConfiguration, _collectionInitializers);
+
+            await _cloudBlobContainer1.Received(1).CreateIfNotExistsAsync();
+            await _cloudBlobContainer2.Received(1).CreateIfNotExistsAsync();
+        }
+
+        [Fact]
+        public void GivenAnInvalidContainerName_WhenInitializing_ThenCheckExceptionIsThrown()
+        {
+            string invalidContainerName = "HelloWorld";
+            Assert.Throws<ArgumentException>(() => new BlobContainerInitializer(invalidContainerName, NullLogger<BlobContainerInitializer>.Instance));
+        }
+    }
+}

src/Microsoft.Health.Blob.UnitTests/Microsoft.Health.Blob.UnitTests.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Azure.Storage.File" Version="10.0.3" />
+    <PackageReference Include="NSubstitute" Version="3.1.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.Health.Blob\Microsoft.Health.Blob.csproj" />
+  </ItemGroup>
+
+</Project>

src/Microsoft.Health.Blob/AssemblyInfo.cs

@@ -0,0 +1,11 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Resources;
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("Microsoft.Health.Blob.UnitTests")]
+[assembly: InternalsVisibleTo("Microsoft.Health.Dicom.Tests.Integration")]
+[assembly: NeutralResourcesLanguage("en-us")]

src/Microsoft.Health.Blob/Configs/BlobContainerConfiguration.cs

@@ -0,0 +1,12 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Blob.Configs
+{
+    public class BlobContainerConfiguration
+    {
+        public string ContainerName { get; set; }
+    }
+}

src/Microsoft.Health.Blob/Configs/BlobDataStoreConfiguration.cs

@@ -0,0 +1,14 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Blob.Configs
+{
+    public class BlobDataStoreConfiguration
+    {
+        public string ConnectionString { get; set; }
+
+        public BlobDataStoreRequestOptions RequestOptions { get; } = new BlobDataStoreRequestOptions();
+    }
+}

src/Microsoft.Health.Blob/Configs/BlobDataStoreRequestOptions.cs

@@ -0,0 +1,18 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Blob.Configs
+{
+    public class BlobDataStoreRequestOptions
+    {
+        public int ExponentialRetryBackoffDeltaInSeconds { get; set; } = 4;
+
+        public int ExponentialRetryMaxAttempts { get; set; } = 6;
+
+        public int ServerTimeoutInMinutes { get; set; } = 2;
+
+        public int ParallelOperationThreadCount { get; set; } = 2;
+    }
+}