Merge pull request #14 from 0sc/develop

add activestorage as dependencies
refactor implementation to leverage more cloudinary sdk methods
update download implementation to support chunked download
bump version to 0.2.0

Gemfile

@@ -5,4 +5,5 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 # Specify your gem's dependencies in active_storage-cloudinary_service.gemspec
 gemspec
 
+gem 'activestorage', '~> 5.2.0', require: false
 gem 'cloudinary', '~> 1.8.2', require: false

Gemfile.lock

@@ -1,29 +1,82 @@
 PATH
   remote: .
   specs:
-    activestorage-cloudinary-service (0.1.0)
+    activestorage-cloudinary-service (0.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    actionpack (5.2.0)
+      actionview (= 5.2.0)
+      activesupport (= 5.2.0)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.0)
+      activesupport (= 5.2.0)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activemodel (5.2.0)
+      activesupport (= 5.2.0)
+    activerecord (5.2.0)
+      activemodel (= 5.2.0)
+      activesupport (= 5.2.0)
+      arel (>= 9.0)
+    activestorage (5.2.0)
+      actionpack (= 5.2.0)
+      activerecord (= 5.2.0)
+      marcel (~> 0.3.1)
+    activesupport (5.2.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
     aws_cf_signer (0.1.3)
+    builder (3.2.3)
     cloudinary (1.8.2)
       aws_cf_signer
       rest-client
     coderay (1.1.2)
+    concurrent-ruby (1.0.5)
+    crass (1.0.4)
     diff-lcs (1.3)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
+    erubi (1.7.1)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
+    i18n (1.0.1)
+      concurrent-ruby (~> 1.0)
+    loofah (2.2.2)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    marcel (0.3.2)
+      mimemagic (~> 0.3.2)
     method_source (0.9.0)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
+    mimemagic (0.3.2)
+    mini_portile2 (2.3.0)
+    minitest (5.11.3)
     netrc (0.11.0)
+    nokogiri (1.8.4)
+      mini_portile2 (~> 2.3.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    rack (2.0.5)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
     rake (10.5.0)
     rest-client (2.0.2)
       http-cookie (>= 1.0.2, < 2.0)
@@ -42,6 +95,9 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.4)
@@ -50,6 +106,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activestorage (~> 5.2.0)
   activestorage-cloudinary-service!
   bundler (~> 1.16)
   cloudinary (~> 1.8.2)

activestorage-cloudinary-service.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'pry', '~> 0.11.3'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.7.0'
-  spec.add_development_dependency 'pry', '~> 0.11.3'
 end

lib/active_storage/service/cloudinary_service.rb

@@ -1,8 +1,12 @@
 require 'cloudinary'
-require 'open-uri'
+require_relative 'download_utils'
 
 module ActiveStorage
+  # Wraps the Cloudinary as an Active Storage service.
+  # See ActiveStorage::Service for the generic API documentation that applies to all services.
   class Service::CloudinaryService < Service
+    include DownloadUtils
+
     # FIXME: implement setup for private resource type
     # FIXME: allow configuration via cloudinary url
     def initialize(cloud_name:, api_key:, api_secret:, **options)
@@ -22,23 +26,27 @@ def upload(key, io, checksum: nil)
     end
 
     # Return the content of the file at the +key+.
-    def download(key)
-      tmp_file = open(url_for_public_id(key))
+    def download(key, &block)
       if block_given?
         instrument :streaming_download, key: key do
-          File.open(tmp_file, 'rb') do |file|
-            while (data = file.read(64.kilobytes))
-              yield data
-            end
-          end
+          source = cloudinary_url_for_key(key)
+          stream_download(source, &block)
         end
       else
         instrument :download, key: key do
-          File.binread tmp_file
+          Cloudinary::Downloader.download(key)
         end
       end
     end
 
+    # Return the partial content in the byte +range+ of the file at the +key+.
+    def download_chunk(key, range)
+      instrument :download_chunk, key: key, range: range do
+        source = cloudinary_url_for_key(key)
+        download_range(source, range)
+      end
+    end
+
     # Delete the file at the +key+.
     def delete(key)
       instrument :delete, key: key do
@@ -49,9 +57,7 @@ def delete(key)
     # Delete files at keys starting with the +prefix+.
     def delete_prefixed(prefix)
       instrument :delete_prefixed, prefix: prefix do
-        find_resources_with_public_id_prefix(prefix).each do |resource|
-          delete_resource_with_public_id(resource['public_id'])
-        end
+        Cloudinary::Api.delete_resources_by_prefix(prefix)
       end
     end
 
@@ -87,9 +93,15 @@ def url_for_direct_upload(key, expires_in:, content_type:, content_length:, chec
           expires_in: expires_in,
           content_type: content_type,
           content_length: content_length,
-          checksum: checksum
+          checksum: checksum,
+          resource_type: 'auto'
         }
-        signed_upload_url_for_public_id(key, options)
+
+        # FIXME: Cloudinary Ruby SDK does't expose an api for signed upload url
+        # The expected url is similar to the private_download_url
+        # with download replaced with upload
+        signed_download_url_for_public_id(key, options)
+          .sub(/download/, 'upload')
       end
     end
 
@@ -108,46 +120,25 @@ def find_resource_with_public_id(public_id)
       Cloudinary::Api.resources_by_ids(public_id).fetch('resources')
     end
 
-    def find_resources_with_public_id_prefix(prefix)
-      Cloudinary::Api.resources(
-        type: :upload,
-        prefix: prefix
-      ).fetch('resources')
-    end
-
     def delete_resource_with_public_id(public_id)
       Cloudinary::Uploader.destroy(public_id)
     end
 
-    def url_for_public_id(public_id)
-      Cloudinary::Api.resource(public_id)['secure_url']
-    end
-
-    # FIXME: Cloudinary Ruby SDK does't expose an api for signed upload url
-    # The expected url is similar to the private_download_url
-    # with download replaced with upload
-    def signed_upload_url_for_public_id(public_id, options)
-      # allow the server to auto detect the resource_type
-      options[:resource_type] ||= 'auto'
-      signed_download_url_for_public_id(public_id, options)
-        .sub(/download/, 'upload')
-    end
-
     def signed_download_url_for_public_id(public_id, options)
-      extension = resource_format(options)
+      extension = resource_format(options[:filename])
       options[:resource_type] ||= resource_type(extension)
 
       Cloudinary::Utils.private_download_url(
-        finalize_public_id(public_id, extension, options),
+        finalize_public_id(public_id, extension, options[:resource_type]),
         extension,
         signed_url_options(options)
       )
     end
 
     # TODO: for assets of type raw,
     # cloudinary request the extension to be part of the public_id
-    def finalize_public_id(public_id, extension, options)
-      return public_id unless options[:resource_type] == 'raw'
+    def finalize_public_id(public_id, extension, resource_type)
+      return public_id unless resource_type == 'raw'
       public_id + '.' + extension
     end
 
@@ -160,13 +151,17 @@ def signed_url_options(options)
       }
     end
 
-    def resource_format(options)
-      extension = options[:filename]&.extension_with_delimiter || ''
+    def resource_format(filename)
+      extension = filename&.extension_with_delimiter || ''
       extension.sub('.', '')
     end
 
     def resource_type(extension)
       Cloudinary::Utils.resource_type_for_format(extension)
     end
+
+    def cloudinary_url_for_key(key)
+      Cloudinary::Utils.cloudinary_url(key)
+    end
   end
 end

lib/active_storage/service/download_utils.rb

@@ -0,0 +1,50 @@
+require 'net/http'
+require 'openssl'
+
+module DownloadUtils
+  def stream_download(source, chunk_size = 5_242_880)
+    url = URI.parse(source)
+    http, req = setup_connection(url)
+
+    content_length = http.request_head(url).content_length
+    upper_limit = content_length + (content_length % chunk_size)
+    offset = 0
+
+    http.start do |agent|
+      while offset < upper_limit
+        lim = (offset + chunk_size)
+        # QUESTION: is it relevant to set the last chunk
+        # to the exact remaining bytes
+        # lim = content_length if lim > content_length
+        req.range = (offset..lim)
+
+        chunk = agent.request(req).body
+        yield chunk.force_encoding(Encoding::BINARY)
+
+        offset += chunk_size + 1
+      end
+    end
+  end
+
+  def download_range(source, range)
+    url = URI.parse(source)
+    http, req = setup_connection(url)
+    req.range = range
+
+    chunk = http.start { |agent| agent.request(req).body }
+    chunk.force_encoding(Encoding::BINARY)
+  end
+
+  private
+
+  def setup_connection(url)
+    http = Net::HTTP.new(url.host, url.port)
+    req = Net::HTTP::Get.new(url.request_uri)
+
+    if url.port == 443
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+    [http, req]
+  end
+end

lib/active_storage/service/version.rb

@@ -1,5 +1,5 @@
 module ActiveStorage
   module CloudinaryService
-    VERSION = '0.1.0'.freeze
+    VERSION = '0.2.0'.freeze
   end
 end