Azure Storage support (#36)

* Microsoft Azure storage support

* Add support for Microsoft Azure Storage

* Comply with the new headers implementation

Gemfile

@@ -14,6 +14,9 @@ gem "httparty"
 
 gem "aws-sdk", "~> 2", require: false
 gem "google-cloud-storage", "~> 1.3", require: false
+# Contains fix to be able to test using StringIO
+gem 'azure-core', git: "https://github.com/dixpac/azure-ruby-asm-core.git"
+gem 'azure-storage', require: false
 
 gem "mini_magick"
 

Gemfile.lock

@@ -1,3 +1,12 @@
+GIT
+  remote: https://github.com/dixpac/azure-ruby-asm-core.git
+  revision: 4403389747f44a94b73e7a7522d1ea11f8b1a266
+  specs:
+    azure-core (0.1.8)
+      faraday (~> 0.9)
+      faraday_middleware (~> 0.10)
+      nokogiri (~> 1.7)
+
 GIT
   remote: https://github.com/rails/rails.git
   revision: 127b475dc251a06942fe0cd2de2e0545cf5ed69f
@@ -79,6 +88,11 @@ GEM
     aws-sdk-resources (2.10.7)
       aws-sdk-core (= 2.10.7)
     aws-sigv4 (1.0.0)
+    azure-storage (0.11.4.preview)
+      azure-core (~> 0.1)
+      faraday (~> 0.9)
+      faraday_middleware (~> 0.10)
+      nokogiri (~> 1.6)
     builder (3.2.3)
     byebug (9.0.6)
     concurrent-ruby (1.0.5)
@@ -88,6 +102,8 @@ GEM
     erubi (1.6.1)
     faraday (0.12.1)
       multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.12.0)
+      faraday (>= 0.7.4, < 1.0)
     globalid (0.4.0)
       activesupport (>= 4.2.0)
     google-api-client (0.13.0)
@@ -201,6 +217,8 @@ PLATFORMS
 DEPENDENCIES
   activestorage!
   aws-sdk (~> 2)
+  azure-core!
+  azure-storage
   bundler (~> 1.15)
   byebug
   google-cloud-storage (~> 1.3)

config/storage_services.yml

@@ -21,6 +21,13 @@ google:
   keyfile: <%= Rails.root.join("path/to/gcs.keyfile") %>
   bucket: your_own_bucket
 
+microsoft:
+  service: Azure
+  path: your_azure_storage_path
+  storage_account_name: your_account_name
+  storage_access_key: <%= Rails.application.secrets.azure[:secret_access_key] %>
+  container: your_container_name
+
 mirror:
   service: Mirror
   primary: local

lib/active_storage/service/azure_service.rb

@@ -0,0 +1,115 @@
+require "active_support/core_ext/numeric/bytes"
+require "azure/storage"
+require "azure/storage/core/auth/shared_access_signature"
+
+# Wraps the Microsoft Azure Storage Blob Service as a Active Storage service.
+# See `ActiveStorage::Service` for the generic API documentation that applies to all services.
+class ActiveStorage::Service::AzureService < ActiveStorage::Service
+  attr_reader :client, :path, :blobs, :container, :signer
+
+  def initialize(path:, storage_account_name:, storage_access_key:, container:)
+    @client = Azure::Storage::Client.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key)
+    @signer = Azure::Storage::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key)
+    @blobs = client.blob_client
+    @container = container
+    @path = path
+  end
+
+  def upload(key, io, checksum: nil)
+    instrument :upload, key, checksum: checksum do
+      begin
+        blobs.create_block_blob(container, key, io, content_md5: checksum)
+      rescue Azure::Core::Http::HTTPError => e
+        raise ActiveStorage::IntegrityError
+      end
+    end
+  end
+
+  def download(key)
+    if block_given?
+      instrument :streaming_download, key do
+        stream(key, &block)
+      end
+    else
+      instrument :download, key do
+        _, io = blobs.get_blob(container, key)
+        io.force_encoding(Encoding::BINARY)
+      end
+    end
+  end
+
+  def delete(key)
+    instrument :delete, key do
+      begin
+        blobs.delete_blob(container, key)
+      rescue Azure::Core::Http::HTTPError
+        false
+      end
+    end
+  end
+
+  def exist?(key)
+    instrument :exist, key do |payload|
+      answer = blob_for(key).present?
+      payload[:exist] = answer
+      answer
+    end
+  end
+
+  def url(key, expires_in:, disposition:, filename:)
+    instrument :url, key do |payload|
+      base_url = url_for(key)
+      generated_url = signer.signed_uri(URI(base_url), false, permissions: "r",
+        expiry: format_expiry(expires_in), content_disposition: "#{disposition}; filename=\"#{filename}\"").to_s
+
+      payload[:url] = generated_url
+
+      generated_url
+    end
+  end
+
+  def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+    instrument :url, key do |payload|
+      base_url = url_for(key)
+      generated_url = signer.signed_uri(URI(base_url), false, permissions: "rw",
+        expiry: format_expiry(expires_in)).to_s
+
+      payload[:url] = generated_url
+
+      generated_url
+    end
+  end
+
+  def headers_for_direct_upload(key, content_type:, checksum:, **)
+    { "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-type" => "BlockBlob" }
+  end
+
+  private
+    def url_for(key)
+      "#{path}/#{container}/#{key}"
+    end
+
+    def blob_for(key)
+      blobs.get_blob_properties(container, key)
+    rescue Azure::Core::Http::HTTPError
+      false
+    end
+
+    def format_expiry(expires_in)
+      expires_in ? Time.now.utc.advance(seconds: expires_in).iso8601 : nil
+    end
+
+    # Reads the object for the given key in chunks, yielding each to the block.
+    def stream(key, options = {}, &block)
+      blob = blob_for(key)
+
+      chunk_size = 5.megabytes
+      offset = 0
+
+      while offset < blob.properties[:content_length]
+        _, io = blobs.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1)
+        yield io
+        offset += chunk_size
+      end
+    end
+end

test/controllers/direct_uploads_controller_test.rb

@@ -67,6 +67,40 @@ class ActiveStorage::GCSDirectUploadsControllerTest < ActionDispatch::Integratio
   puts "Skipping GCS Direct Upload tests because no GCS configuration was supplied"
 end
 
+if SERVICE_CONFIGURATIONS[:azure]
+  class ActiveStorage::AzureDirectUploadsControllerTest < ActionDispatch::IntegrationTest
+    setup do
+      @config = SERVICE_CONFIGURATIONS[:azure]
+
+      @old_service = ActiveStorage::Blob.service
+      ActiveStorage::Blob.service = ActiveStorage::Service.configure(:azure, SERVICE_CONFIGURATIONS)
+    end
+
+    teardown do
+      ActiveStorage::Blob.service = @old_service
+    end
+
+    test "creating new direct upload" do
+      checksum = Digest::MD5.base64digest("Hello")
+
+      post rails_direct_uploads_url, params: { blob: {
+        filename: "hello.txt", byte_size: 6, checksum: checksum, content_type: "text/plain" } }
+
+      @response.parsed_body.tap do |details|
+        assert_equal ActiveStorage::Blob.find(details["id"]), ActiveStorage::Blob.find_signed(details["signed_id"])
+        assert_equal "hello.txt", details["filename"]
+        assert_equal 6, details["byte_size"]
+        assert_equal checksum, details["checksum"]
+        assert_equal "text/plain", details["content_type"]
+        assert_match %r{#{@config[:storage_account_name]}\.blob\.core\.windows\.net/#{@config[:container]}}, details["direct_upload"]["url"]
+        assert_equal({ "Content-Type" => "text/plain", "Content-MD5" => checksum, "x-ms-blob-type" => "BlockBlob" }, details["direct_upload"]["headers"])
+      end
+    end
+  end
+else
+  puts "Skipping Azure Direct Upload tests because no Azure configuration was supplied"
+end
+
 class ActiveStorage::DiskDirectUploadsControllerTest < ActionDispatch::IntegrationTest
   test "creating new direct upload" do
     checksum = Digest::MD5.base64digest("Hello")

test/service/azure_service_test.rb

@@ -0,0 +1,14 @@
+require "service/shared_service_tests"
+require "httparty"
+require "uri"
+
+if SERVICE_CONFIGURATIONS[:azure]
+  class ActiveStorage::Service::AzureServiceTest < ActiveSupport::TestCase
+    SERVICE = ActiveStorage::Service.configure(:azure, SERVICE_CONFIGURATIONS)
+
+    include ActiveStorage::Service::SharedServiceTests
+  end
+
+else
+  puts "Skipping Azure Storage Service tests because no Azure configuration was supplied"
+end

test/service/configurations-example.yml

@@ -22,3 +22,10 @@ gcs:
   }
   project:
   bucket:
+
+azure:
+  service: Azure
+  path: ""
+  storage_account_name: ""
+  storage_access_key: ""
+  container: ""