Do not upload files with identical etags

lib/nanoc/extra/deployers/fog.rb

@@ -71,19 +71,15 @@ def run
       end
       keys_to_destroy = files.all.map(&:key)
       keys_to_invalidate = []
+      etags = read_etags(files)
 
       # Upload all the files in the output folder to the clouds
       puts 'Uploading local files'
       FileUtils.cd(src) do
         files = Dir['**/*'].select { |f| File.file?(f) }
         files.each do |file_path|
           key = path ? File.join(path, file_path) : file_path
-          directory.files.create(
-            key: key,
-            body: File.open(file_path),
-            public: true)
-          keys_to_destroy.delete(key)
-          keys_to_invalidate.push(key)
+          upload(key, file_path, etags, keys_to_destroy, keys_to_invalidate, directory)
         end
       end
 
@@ -113,6 +109,44 @@ def run
 
     private
 
+    def upload(key, file_path, etags, keys_to_destroy, keys_to_invalidate, dir)
+      keys_to_destroy.delete(key)
+
+      return unless needs_upload?(key, file_path, etags)
+
+      dir.files.create(
+        key: key,
+        body: File.open(file_path),
+        public: true)
+      keys_to_invalidate.push(key)
+    end
+
+    def needs_upload?(key, file_path, etags)
+      remote_etag = etags[key]
+      return true if remote_etag.nil?
+
+      local_etag = calc_local_etag(file_path)
+      remote_etag != local_etag
+    end
+
+    def read_etags(files)
+      case config[:provider]
+      when 'aws'
+        files.each_with_object({}) do |file, etags|
+          etags[file.key] = file.etag
+        end
+      else
+        {}
+      end
+    end
+
+    def calc_local_etag(file_path)
+      case config[:provider]
+      when 'aws'
+        Digest::MD5.file(file_path).hexdigest
+      end
+    end
+
     # Prints the given message on stderr and exits.
     def error(msg)
       raise RuntimeError.new(msg)

test/extra/deployers/test_fog.rb

@@ -126,4 +126,151 @@ def test_run_delete_stray
       assert_equal 'I am a dog!', File.read('mylocalcloud/mybucket/bark')
     end
   end
+
+  def test_upload
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'aws')
+
+      key_old = '__old'
+      key_same = '__same'
+      key_new = '__new'
+
+      File.write(key_same, 'hallo')
+      File.write(key_new, 'hallo new')
+
+      etags = {
+        key_same => '598d4c200461b81522a3328565c25f7c',
+        key_new => '598d4c200461b81522a3328565c25f7c',
+      }
+
+      keys_to_destroy = [key_old, key_same, key_new]
+      keys_to_invalidate = []
+
+      s3_files = mock
+      s3_files.stubs(:create)
+      s3_directory = mock
+      s3_directory.stubs(:files).returns(s3_files)
+
+      # key_same
+      refute fog.send(:needs_upload?, key_same, key_same, etags)
+      fog.send(
+        :upload, key_same, key_same, etags, keys_to_destroy, keys_to_invalidate, s3_directory)
+
+      assert_equal([key_old, key_new], keys_to_destroy)
+      assert_equal([], keys_to_invalidate)
+
+      # key_new
+      assert fog.send(:needs_upload?, key_new, key_new, etags)
+      fog.send(
+        :upload, key_new, key_new, etags, keys_to_destroy, keys_to_invalidate, s3_directory)
+
+      assert_equal([key_old], keys_to_destroy)
+      assert_equal([key_new], keys_to_invalidate)
+    end
+  end
+
+  def test_read_etags_with_local_provider
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'local')
+
+      files = [
+        mock('file_a'),
+        mock('file_b'),
+      ]
+
+      assert_equal({}, fog.send(:read_etags, files))
+    end
+  end
+
+  def test_read_etags_with_aws_provider
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'aws')
+
+      files = [
+        mock('file_a', key: 'key_a', etag: 'etag_a'),
+        mock('file_b', key: 'key_b', etag: 'etag_b'),
+      ]
+
+      expected = {
+        'key_a' => 'etag_a',
+        'key_b' => 'etag_b',
+      }
+
+      assert_equal(expected, fog.send(:read_etags, files))
+    end
+  end
+
+  def test_calc_local_etag_with_local_provider
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'local')
+
+      file_path = 'blah.tmp'
+      File.write(file_path, 'hallo')
+
+      assert_nil fog.send(:calc_local_etag, file_path)
+    end
+  end
+
+  def test_calc_local_etag_with_aws_provider
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'aws')
+
+      file_path = 'blah.tmp'
+      File.write(file_path, 'hallo')
+
+      assert_equal(
+        '598d4c200461b81522a3328565c25f7c',
+        fog.send(:calc_local_etag, file_path))
+    end
+  end
+
+  def test_needs_upload_with_missing_remote_etag
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'aws')
+
+      file_path = 'blah.tmp'
+      File.write(file_path, 'hallo')
+
+      key = 'some_key'
+      etags = {}
+
+      assert fog.send(:needs_upload?, key, file_path, etags)
+    end
+  end
+
+  def test_needs_upload_with_different_etags
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'aws')
+
+      file_path = 'blah.tmp'
+      File.write(file_path, 'hallo')
+
+      key = 'some_key'
+      etags = { key => 'some_etag' }
+
+      assert fog.send(:needs_upload?, key, file_path, etags)
+    end
+  end
+
+  def test_needs_upload_with_identical_etags
+    if_have 'fog' do
+      fog = Nanoc::Extra::Deployers::Fog.new(
+        'output/', provider: 'aws')
+
+      file_path = 'blah.tmp'
+      File.write(file_path, 'hallo')
+
+      key = 'some_key'
+      etags = { key => '598d4c200461b81522a3328565c25f7c' }
+
+      refute fog.send(:needs_upload?, key, file_path, etags)
+    end
+  end
 end