Allow a Fog::Storage uploader to be passed in for uploading of genera…

…ted tiles rather than local storage

CHANGELOG.md

@@ -1,3 +1,7 @@
+### Next
+
+* [#5](https://github.com/dblock/dzt/pull/5) Allow a Fog::Storage uploader to be passed in for uploading of generated tiles rather than local storage
+
 ### 0.1.0 (3/16/2014)
 
 * Initial public release - [@dblock](https://github.com/dblock).

Gemfile

@@ -4,3 +4,4 @@ gemspec
 
 gem "rspec"
 gem "rake"
+gem "fog"

bin/dzt

@@ -15,9 +15,12 @@ command :slice do |c|
     if args.length < 1
       raise 'You must specify an image file to slice.'
     end
+    storage = DZT::FileStorage.new(
+      destination: options[:output]
+    )
     tiler = DZT::Tiler.new(
       source: args[0],
-      destination: options[:output]
+      storage: storage
     )
     tiler.slice! do |path|
       puts path

lib/dzt/file_storage.rb

@@ -0,0 +1,29 @@
+module DZT
+  class FileStorage
+    #
+    # @param overwrite: If true, overwrites existing tiles, defaults to false.
+    # @param destination: Full directory in which to output tiles, defaults to 'tiles' in the current dir.
+    #
+    def initialize(options = {})
+      @overwrite = options[:overwrite] || false
+      @store_path = options[:destination] || File.join(Dir.pwd, 'tiles')
+    end
+
+    def exists?
+      ! @overwrite && File.directory?(@store_path) && ! Dir["@{@store_path}/*"].empty?
+    end
+
+    def storage_location(level)
+      File.join(@store_path, level.to_s)
+    end
+
+    def mkdir(path)
+      FileUtils.mkdir_p(path)
+    end
+
+    def write(file, dest, options = {})
+      quality = options[:quality]
+      file.write(dest) { @quality = quality if quality }
+    end
+  end
+end

lib/dzt/s3_storage.rb

@@ -0,0 +1,47 @@
+module DZT
+  class S3Storage
+    #
+    # @param s3_acl: ACL to use for storing, defaults to 'public-read'.
+    # @param s3_bucket: Bucket to store tiles.
+    # @param s3_key: Key to prefix stored files.
+    # @param aws_id: AWS Id.
+    # @param aws_secret: AWS Secret.
+    #
+    def initialize(options = {})
+      @s3_acl = options[:s3_acl] || 'public-read'
+      @s3_bucket = options[:s3_bucket]
+      @s3_key = options[:s3_key]
+      @s3_id = options[:aws_id]
+      @s3_secret = options[:aws_secret]
+    end
+
+    def s3
+      @s3 ||= Fog::Storage.new(
+        provider: 'AWS',
+        aws_access_key_id: @s3_id,
+        aws_secret_access_key: @s3_secret
+      )
+    end
+
+    # Currently does not supporting checking S3 fo overwritten files
+    def exists?
+      false
+    end
+
+    def storage_location(level)
+      "#{@s3_key}/#{level.to_s}"
+    end
+
+    # no-op
+    def mkdir(path)
+    end
+
+    def write(file, dest, options = {})
+      quality = options[:quality]
+      @s3.put_object(@s3_bucket, dest, file.to_blob { @quality = quality if quality },
+        'Content-Type' => file.mime_type,
+        'x-amz-acl' => @s3_acl
+      )
+    end
+  end 
+end

lib/dzt/tiler.rb

@@ -1,4 +1,6 @@
 # Deep Zoom module for generating Deep Zoom (DZI) tiles from a source image
+require_relative 'file_storage'
+require_relative 's3_storage'
 module DZT
   class Tiler
     # Defaults
@@ -14,32 +16,31 @@ class Tiler
     # @param format Format for output tiles (default: "jpg")
     # @param size Size, in pixels, for tile squares (default: 512)
     # @param overlap Size, in pixels, of the overlap between tiles (default: 2)
-    # @param overwrite If true, overwrites existing tiles (default: false)
-    # @param destination: Full directory in which to output tiles.
+    # @param storage Either an instance of S3Storage or FileStorage
     #
     def initialize(options)
       @tile_source = options[:source]
       raise "Missing options[:source]." unless @tile_source
+
       @tile_source = Magick::Image.read(@tile_source)[0] if @tile_source.is_a?(String)
       @tile_size = options[:size] || DEFAULT_TILE_SIZE
       @tile_overlap = options[:overlap] || DEFAULT_TILE_OVERLAP
       @tile_format  = options[:format] || DEFAULT_TILE_FORMAT
+
       @max_tiled_height = @tile_source.rows
       @max_tiled_width = @tile_source.columns
+
       @tile_quality = options[:quality] || DEFAULT_QUALITY
       @overwrite = options[:overwrite] || false
-      @destination = options[:destination] || File.join(Dir.pwd, "tiles")
+      @storage = options[:storage]
     end
 
     ##
     # Generates the DZI-formatted tiles and sets necessary metadata on this object.
     # Uses a default tile size of 512 pixels, with a default overlap of 2 pixel.
     ##
     def slice!(&block)
-      if ! @overwrite && File.directory?(@destination) && ! Dir["@{@destination}/*"].empty?
-        raise "Output directory #{@destination} already exists!"
-        @overwrite ? Rails.logger.warn(msg) : raise(msg)
-      end
+      raise "Output #{@destination} already exists!" if ! @overwrite && @storage.exists?
 
       image = @tile_source.dup
       orig_width, orig_height = image.columns, image.rows
@@ -48,11 +49,10 @@ def slice!(&block)
       max_level(orig_width, orig_height).downto(0) do |level|
         width, height = image.columns, image.rows
 
-        current_level_dir = File.join(@destination, level.to_s)
-        FileUtils.mkdir_p(current_level_dir)
-
+        current_level_storage_dir = @storage.storage_location(level)
+        @storage.mkdir(current_level_storage_dir)
         if block_given?
-          yield current_level_dir
+          yield current_level_storage_dir
         end
 
         # iterate over columns
@@ -61,7 +61,7 @@ def slice!(&block)
           # iterate over rows
           y, row_count = 0, 0
           while y < height
-            dest_path = File.join(current_level_dir, "#{col_count}_#{row_count}.#{@tile_format}")
+            dest_path = File.join(current_level_storage_dir, "#{col_count}_#{row_count}.#{@tile_format}")
             tile_width, tile_height = tile_dimensions(x, y, @tile_size, @tile_overlap)
 
             save_cropped_image(image, dest_path, x, y, tile_width, tile_height, @tile_quality)
@@ -120,7 +120,7 @@ def save_cropped_image(src, dest, x, y, width, height, quality = 75)
       # The crop method retains the offset information in the cropped image.
       # To reset the offset data, adding true as the last argument to crop.
       cropped = img.crop(x, y, width, height, true)
-      cropped.write(dest) { @quality = quality }
+      @storage.write(cropped, dest, quality: quality)
     end
   end
 end

spec/dzt/dzt_spec.rb

@@ -8,19 +8,19 @@
   describe "#help" do
     it "displays help" do
       help = `"#{@binary}" help`
-      help.should include "dzt - Tile images into deep-zoom tiles"
+      expect(help).to include "dzt - Tile images into deep-zoom tiles"
     end
   end
   describe "#slice" do
     it "slices an image" do
       goya = File.join(@fixtures_dir, "francisco-jose-de-goya-y-lucientes-senora-sabasa-garcia.jpg")
       Dir.mktmpdir do |tmpdir|
         `"#{@binary}" slice "#{goya}" --output #{tmpdir}`
-        Dir["#{tmpdir}/*"].map { |dir| dir.split("/").last.to_i }.sort.should == (0..12).to_a
+        expect(Dir["#{tmpdir}/*"].map { |dir| dir.split("/").last.to_i }.sort).to eq((0..12).to_a)
         # center
         image = Magick::Image::read("#{tmpdir}/11/1_1.jpg").first
-        image.columns.should == 512
-        image.rows.should == 512
+        expect(image.columns).to eq(512)
+        expect(image.rows).to eq(512)
       end
     end
   end

spec/dzt/tiler_spec.rb

@@ -1,25 +1,65 @@
 require 'spec_helper'
+require 'RMagick'
+include Magick
 
 describe DZT::Tiler do
   before :each do
     @fixtures_dir = File.expand_path(File.join(__FILE__, '../../fixtures'))
   end
-  it "slices an image" do
-    Dir.mktmpdir do |tmpdir|
-      tiler = DZT::Tiler.new(
+  context 'storing files locally' do
+    it "slices an image and stores files" do
+      Dir.mktmpdir do |tmpdir|
+        storage = DZT::FileStorage.new(destination: tmpdir)
+        tiler = DZT::Tiler.new(
+          source: File.join(@fixtures_dir, "francisco-jose-de-goya-y-lucientes-senora-sabasa-garcia.jpg"),
+          storage: storage
+        )
+        tiler.slice!
+        expect(Dir["#{tmpdir}/*"].map { |dir| dir.split("/").last.to_i }.sort).to eq((0..12).to_a)
+        # center
+        image = Magick::Image::read("#{tmpdir}/11/1_1.jpg").first
+        expect(image.columns).to eq(512)
+        expect(image.rows).to eq(512)
+        # edge
+        image = Magick::Image::read("#{tmpdir}/11/2_2.jpg").first
+        expect(image.columns).to eq(168)
+        expect(image.rows).to eq(443)
+      end
+    end
+  end
+  context 'uploading resultant files to S3' do
+    before :each do
+      Fog.mock!
+      Fog::Mock.reset
+      storage = DZT::S3Storage.new(
+        s3_acl: 'public-read',
+        s3_bucket: 'tiled-images',
+        s3_key: 'dztiles',
+        aws_id: 'id',
+        aws_secret: 'secret'
+      )
+      @bucket = storage.s3.directories.create(key: 'tiled-images')
+      @tiler = DZT::Tiler.new(
         source: File.join(@fixtures_dir, "francisco-jose-de-goya-y-lucientes-senora-sabasa-garcia.jpg"),
-        destination: tmpdir
+        storage: storage
       )
-      tiler.slice!
-      Dir["#{tmpdir}/*"].map { |dir| dir.split("/").last.to_i }.sort.should == (0..12).to_a
-      # center
-      image = Magick::Image::read("#{tmpdir}/11/1_1.jpg").first
-      image.columns.should == 512
-      image.rows.should == 512
-      # edge
-      image = Magick::Image::read("#{tmpdir}/11/2_2.jpg").first
-      image.columns.should == 168
-      image.rows.should == 443
+    end
+    it 'slices the images' do
+      @tiler.slice!
+      file = @bucket.files.select { |f| f.key == 'dztiles/11/1_1.jpg' }.first
+      image = Image.from_blob(file.body).first
+      expect(image.columns).to eq(512)
+      expect(image.rows).to eq(512)
+      file = @bucket.files.select { |f| f.key == 'dztiles/11/2_2.jpg' }.first
+      image = Image.from_blob(file.body).first
+      expect(image.columns).to eq(168)
+      expect(image.rows).to eq(443)
+    end
+    it 'stores the files properly' do
+      expect_any_instance_of(Fog::Storage::AWS::Mock).to receive(:put_object).at_least(53).times do |*args|
+        expect(args.last).to eq("Content-Type" => "image/jpeg", "x-amz-acl" => "public-read")
+      end
+      @tiler.slice!
     end
   end
 end

spec/dzt/version_spec.rb

@@ -2,6 +2,6 @@
 
 describe DZT do
   it "has a version" do
-    DZT::VERSION.should_not be_nil
+    expect(DZT::VERSION).to_not be_nil
   end
 end

spec/spec_helper.rb

@@ -4,3 +4,4 @@
 require 'rspec'
 require 'tmpdir'
 require 'dzt'
+require 'fog'