Detect asset changes by saving source_digests in manifest.yml, and speed up non-digest assets

actionpack/lib/sprockets/assets.rake

@@ -8,9 +8,9 @@ namespace :assets do
     args << "--trace" if Rake.application.options.trace
     if $0 =~ /rake\.bat\Z/i
       Kernel.exec $0, *args
-    else  
+    else
       fork ? ruby(*args) : Kernel.exec(FileUtils::RUBY, *args)
-    end    
+    end
   end
 
   # We are currently running with no explicit bundler group
@@ -43,34 +43,50 @@ namespace :assets do
       config = Rails.application.config
       config.assets.compile = true
       config.assets.digest  = digest unless digest.nil?
-      config.assets.digests = {}
-
-      env      = Rails.application.assets
-      target   = File.join(Rails.public_path, config.assets.prefix)
-      compiler = Sprockets::StaticCompiler.new(env,
-                                               target,
-                                               config.assets.precompile,
-                                               :manifest_path => config.assets.manifest,
-                                               :digest => config.assets.digest,
-                                               :manifest => digest.nil?)
-      compiler.compile
+      config.assets.digests ||= {}
+      config.assets.source_digests ||= {}
+
+      env    = Rails.application.assets
+      target = File.join(::Rails.public_path, config.assets.prefix)
+
+      # If processing non-digest assets, and compiled digest files are
+      # present, then generate non-digest assets from existing assets.
+      # It is assumed that `assets:precompile:nondigest` won't be run manually
+      # if assets have been previously compiled with digests.
+      if !config.assets.digest && config.assets.digests.any?
+        generator = Sprockets::StaticNonDigestGenerator.new(env, target, config.assets.precompile,
+          :digests   => config.assets.digests
+        )
+        generator.generate
+      else
+        compiler = Sprockets::StaticCompiler.new(env, target, config.assets.precompile,
+          :digest         => config.assets.digest,
+          :manifest       => digest.nil?,
+          :manifest_path  => config.assets.manifest,
+          :digests        => config.assets.digests,
+          :source_digests => config.assets.source_digests
+        )
+        compiler.compile
+      end
     end
 
-    task :all do
-      Rake::Task["assets:precompile:primary"].invoke
-      # We need to reinvoke in order to run the secondary digestless
-      # asset compilation run - a fresh Sprockets environment is
-      # required in order to compile digestless assets as the
-      # environment has already cached the assets on the primary
-      # run.
-      ruby_rake_task("assets:precompile:nondigest", false) if Rails.application.config.assets.digest
+    task :all => ["assets:cache:clean"] do
+      internal_precompile
+      internal_precompile(false) if ::Rails.application.config.assets.digest
+
+      # Other gems may want to add hooks to run after the 'assets:precompile:***' tasks.
+      # Since we aren't running separate rake tasks anymore,
+      # we need to manually invoke any extra actions.
+      %w(primary nondigest).each do |asset_type|
+        Rake::Task["assets:precompile:#{asset_type}"].actions[1..-1].each &:call
+      end
     end
 
-    task :primary => ["assets:environment", "tmp:cache:clear"] do
+    task :primary => ["assets:cache:clean"] do
       internal_precompile
     end
 
-    task :nondigest => ["assets:environment", "tmp:cache:clear"] do
+    task :nondigest => ["assets:cache:clean"] do
       internal_precompile(false)
     end
   end
@@ -81,18 +97,24 @@ namespace :assets do
   end
 
   namespace :clean do
-    task :all => ["assets:environment", "tmp:cache:clear"] do
-      config = Rails.application.config
-      public_asset_path = File.join(Rails.public_path, config.assets.prefix)
+    task :all => ["assets:cache:clean"] do
+      config = ::Rails.application.config
+      public_asset_path = File.join(::Rails.public_path, config.assets.prefix)
       rm_rf public_asset_path, :secure => true
     end
   end
 
+  namespace :cache do
+    task :clean => ['tmp:create', "assets:environment"] do
+      ::Rails.application.assets.cache.clear
+    end
+  end
+
   task :environment do
-    if Rails.application.config.assets.initialize_on_precompile
+    if ::Rails.application.config.assets.initialize_on_precompile
       Rake::Task["environment"].invoke
     else
-      Rails.application.initialize!(:assets)
+      ::Rails.application.initialize!(:assets)
       Sprockets::Bootstrap.new(Rails.application).run
     end
   end

actionpack/lib/sprockets/helpers/rails_helper.rb

@@ -10,7 +10,7 @@ def asset_paths
         @asset_paths ||= begin
           paths = RailsHelper::AssetPaths.new(config, controller)
           paths.asset_environment = asset_environment
-          paths.asset_digests     = asset_digests
+          paths.digests      = digests
           paths.compile_assets    = compile_assets?
           paths.digest_assets     = digest_assets?
           paths
@@ -95,7 +95,7 @@ def asset_prefix
         Rails.application.config.assets.prefix
       end
 
-      def asset_digests
+      def digests
         Rails.application.config.assets.digests
       end
 
@@ -115,7 +115,7 @@ def asset_environment
       end
 
       class AssetPaths < ::ActionView::AssetPaths #:nodoc:
-        attr_accessor :asset_environment, :asset_prefix, :asset_digests, :compile_assets, :digest_assets
+        attr_accessor :asset_environment, :asset_prefix, :digests, :compile_assets, :digest_assets
 
         class AssetNotPrecompiledError < StandardError; end
 
@@ -129,7 +129,7 @@ def asset_for(source, ext)
         end
 
         def digest_for(logical_path)
-          if digest_assets && asset_digests && (digest = asset_digests[logical_path])
+          if digest_assets && digests && (digest = digests[logical_path])
             return digest
           end
 

actionpack/lib/sprockets/railtie.rb

@@ -7,6 +7,7 @@ module Sprockets
   autoload :LazyCompressor, "sprockets/compressors"
   autoload :NullCompressor, "sprockets/compressors"
   autoload :StaticCompiler, "sprockets/static_compiler"
+  autoload :StaticNonDigestGenerator, "sprockets/static_non_digest_generator"
 
   # TODO: Get rid of config.assets.enabled
   class Railtie < ::Rails::Railtie
@@ -32,15 +33,11 @@ class Railtie < ::Rails::Railtie
         end
       end
 
-      if config.assets.manifest
-        path = File.join(config.assets.manifest, "manifest.yml")
-      else
-        path = File.join(Rails.public_path, config.assets.prefix, "manifest.yml")
-      end
-
-      if File.exist?(path)
-        config.assets.digests = YAML.load_file(path)
-      end
+      manifest_dir = config.assets.manifest || File.join(Rails.public_path, config.assets.prefix)
+      digests_manifest = File.join(manifest_dir, "manifest.yml")
+      sources_manifest = File.join(manifest_dir, "sources_manifest.yml")
+      config.assets.digests        = (File.exist?(digests_manifest) && YAML.load_file(digests_manifest)) || {}
+      config.assets.source_digests = (File.exist?(sources_manifest) && YAML.load_file(sources_manifest)) || {}
 
       ActiveSupport.on_load(:action_view) do
         include ::Sprockets::Helpers::RailsHelper

actionpack/lib/sprockets/static_compiler.rb

@@ -8,27 +8,82 @@ def initialize(env, target, paths, options = {})
       @env = env
       @target = target
       @paths = paths
-      @digest = options.key?(:digest) ? options.delete(:digest) : true
-      @manifest = options.key?(:manifest) ? options.delete(:manifest) : true
+      @digest = options.fetch(:digest, true)
+      @manifest = options.fetch(:manifest, true)
       @manifest_path = options.delete(:manifest_path) || target
+
+      @current_source_digests = options.fetch(:source_digests, {})
+      @current_digests   = options.fetch(:digests,   {})
+
+      @digests = {}
+      @source_digests = {}
     end
 
     def compile
-      manifest = {}
+      start_time = Time.now.to_f
+
+      # Run asset compilation
+      process_assets
+
+      # Encode all filenames & digests as UTF-8 for Ruby 1.9,
+      # otherwise YAML dumps other string encodings as !binary
+      if RUBY_VERSION.to_f >= 1.9
+        @source_digests = encode_hash_as_utf8 @source_digests
+        @digests = encode_hash_as_utf8 @digests
+      end
+
+      if @manifest
+        write_manifest(@digests, @source_digests)
+      end
+
+      # Store digests in Rails config. (Important if non-digest is run after primary)
+      config = ::Rails.application.config
+      config.assets.digests = @digests
+      config.assets.source_digests = @source_digests
+
+      elapsed_time = ((Time.now.to_f - start_time) * 1000).to_i
+      env.logger.debug "Processed #{'non-' unless @digest}digest assets in #{elapsed_time}ms"
+    end
+
+    # Compiles assets if their source digests haven't changed
+    def process_assets
       env.each_logical_path(paths) do |logical_path|
-        if asset = env.find_asset(logical_path)
-          digest_path = write_asset(asset)
-          manifest[asset.logical_path] = digest_path
-          manifest[aliased_path_for(asset.logical_path)] = digest_path
+        # Fetch asset without any processing or compression,
+        # to calculate a digest of the concatenated source files
+        asset = env.find_asset(logical_path, :process => false)
+
+        @source_digests[logical_path] = asset.digest
+
+        # Recompile if digest has changed or compiled digest file is missing
+        current_digest_file = @current_digests[logical_path]
+
+        if @source_digests[logical_path] != @current_source_digests[logical_path] ||
+           !(current_digest_file && File.exists?("#{@target}/#{current_digest_file}"))
+
+          if asset = env.find_asset(logical_path)
+            digest_path = write_asset(asset)
+            @digests[asset.logical_path] = digest_path
+            @digests[aliased_path_for(asset.logical_path)] = digest_path
+          end
+        else
+          # Set asset file from manifest.yml
+          digest_path = @current_digests[logical_path]
+          @digests[logical_path] = digest_path
+          @digests[aliased_path_for(logical_path)] = digest_path
+
+          env.logger.debug "Not compiling #{logical_path}, sources digest has not changed " <<
+                           "(#{@source_digests[logical_path][0...7]})"
         end
       end
-      write_manifest(manifest) if @manifest
     end
 
-    def write_manifest(manifest)
+    def write_manifest(digests, source_digests)
       FileUtils.mkdir_p(@manifest_path)
       File.open("#{@manifest_path}/manifest.yml", 'wb') do |f|
-        YAML.dump(manifest, f)
+        YAML.dump(digests, f)
+      end
+      File.open("#{@manifest_path}/sources_manifest.yml", 'wb') do |f|
+        YAML.dump(source_digests, f)
       end
     end
 
@@ -41,10 +96,15 @@ def write_asset(asset)
       end
     end
 
+
     def path_for(asset)
       @digest ? asset.digest_path : asset.logical_path
     end
 
+    def encode_hash_as_utf8(hash)
+      Hash[*hash.map {|k,v| [k.encode("UTF-8"), v.encode("UTF-8")] }.flatten]
+    end
+
     def aliased_path_for(logical_path)
       if File.basename(logical_path).start_with?('index')
         logical_path.sub(/\/index([^\/]+)$/, '\1')

actionpack/lib/sprockets/static_non_digest_generator.rb

@@ -0,0 +1,100 @@
+require 'fileutils'
+
+module Sprockets
+  class StaticNonDigestGenerator
+
+    DIGEST_REGEX = /-([0-9a-f]{32})\./
+
+    attr_accessor :env, :target, :paths
+
+    def initialize(env, target, paths, options = {})
+      @env = env
+      @target = target
+      @paths = paths
+      @digests = options.fetch(:digests, {})
+
+      # Parse digests from digests hash
+      @asset_digests = Hash[*@digests.map {|file, digest_file|
+        [file, digest_file[DIGEST_REGEX, 1]]
+      }.flatten]
+    end
+
+
+    # Generate non-digest assets by making a copy of the digest asset,
+    # with digests stripped from js and css. The new files are also gzipped.
+    # Other assets are copied verbatim.
+    def generate
+      start_time = Time.now.to_f
+
+      env.each_logical_path(paths) do |logical_path|
+        digest_path      = @digests[logical_path]
+        abs_digest_path  = "#{@target}/#{digest_path}"
+        abs_logical_path = "#{@target}/#{logical_path}"
+
+        mtime = File.mtime(abs_digest_path)
+
+        # Remove known digests from css & js
+        if abs_digest_path.match(/\.(?:js|css)$/)
+          asset_body = File.read(abs_digest_path)
+
+          # Find all hashes in the asset body with a leading '-'
+          asset_body.gsub!(DIGEST_REGEX) do |match|
+            # Only remove if known digest
+            $1.in?(@asset_digests.values) ? '.' : match
+          end
+
+          # Write non-digest file
+          File.open abs_logical_path, 'w' do |f|
+            f.write asset_body
+          end
+          # Set modification and access times
+          File.utime(File.atime(abs_digest_path), mtime, abs_logical_path)
+
+          # Also write gzipped asset
+          File.open("#{abs_logical_path}.gz", 'wb') do |f|
+            gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION)
+            gz.mtime = mtime.to_i
+            gz.write asset_body
+            gz.close
+          end
+
+          env.logger.debug "Stripped digests, copied to #{logical_path}, and created gzipped asset"
+
+        else
+          # Otherwise, treat file as binary and copy it.
+          # Ignore paths that have no digests, such as READMEs
+          unless abs_digest_path == abs_logical_path
+            FileUtils.cp_r abs_digest_path, abs_logical_path, :remove_destination => true
+            env.logger.debug "Copied binary asset to #{logical_path}"
+
+            # Copy gzipped asset if exists
+            if File.exist? "#{abs_digest_path}.gz"
+              FileUtils.cp_r "#{abs_digest_path}.gz", "#{abs_logical_path}.gz", :remove_destination => true
+              env.logger.debug "Copied gzipped asset to #{logical_path}.gz"
+            end
+          end
+        end
+      end
+
+
+      elapsed_time = ((Time.now.to_f - start_time) * 1000).to_i
+      env.logger.debug "Generated non-digest assets in #{elapsed_time}ms"
+    end
+
+    private
+
+    def compile_path?(logical_path)
+      paths.each do |path|
+        case path
+        when Regexp
+          return true if path.match(logical_path)
+        when Proc
+          return true if path.call(logical_path)
+        else
+          return true if File.fnmatch(path.to_s, logical_path)
+        end
+      end
+      false
+    end
+  end
+end