Detect asset changes by saving source_digests in manifest.yml, and speed up non-digest assets #7866

Closed
wants to merge 1 commit into from
View
78 actionpack/lib/sprockets/assets.rake
@@ -8,9 +8,9 @@ namespace :assets do
args << "--trace" if Rake.application.options.trace
if $0 =~ /rake\.bat\Z/i
Kernel.exec $0, *args
- else
+ else
fork ? ruby(*args) : Kernel.exec(FileUtils::RUBY, *args)
- end
+ end
end
# We are currently running with no explicit bundler group
@@ -43,34 +43,50 @@ namespace :assets do
config = Rails.application.config
config.assets.compile = true
config.assets.digest = digest unless digest.nil?
- config.assets.digests = {}
-
- env = Rails.application.assets
- target = File.join(Rails.public_path, config.assets.prefix)
- compiler = Sprockets::StaticCompiler.new(env,
- target,
- config.assets.precompile,
- :manifest_path => config.assets.manifest,
- :digest => config.assets.digest,
- :manifest => digest.nil?)
- compiler.compile
+ config.assets.digests ||= {}
+ config.assets.source_digests ||= {}
+
+ env = Rails.application.assets
+ target = File.join(::Rails.public_path, config.assets.prefix)
+
+ # If processing non-digest assets, and compiled digest files are
+ # present, then generate non-digest assets from existing assets.
+ # It is assumed that `assets:precompile:nondigest` won't be run manually
+ # if assets have been previously compiled with digests.
+ if !config.assets.digest && config.assets.digests.any?
+ generator = Sprockets::StaticNonDigestGenerator.new(env, target, config.assets.precompile,
+ :digests => config.assets.digests
+ )
+ generator.generate
+ else
+ compiler = Sprockets::StaticCompiler.new(env, target, config.assets.precompile,
+ :digest => config.assets.digest,
+ :manifest => digest.nil?,
+ :manifest_path => config.assets.manifest,
+ :digests => config.assets.digests,
+ :source_digests => config.assets.source_digests
+ )
+ compiler.compile
+ end
end
- task :all do
- Rake::Task["assets:precompile:primary"].invoke
- # We need to reinvoke in order to run the secondary digestless
- # asset compilation run - a fresh Sprockets environment is
- # required in order to compile digestless assets as the
- # environment has already cached the assets on the primary
- # run.
- ruby_rake_task("assets:precompile:nondigest", false) if Rails.application.config.assets.digest
+ task :all => ["assets:cache:clean"] do
@jeremy
Ruby on Rails member
jeremy added a note Oct 7, 2012

See comment above -- always cleaning before generating assets means we we're orphaning references to old assets.

Asset cleaning has been removed from the pull request, please see above

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ internal_precompile
+ internal_precompile(false) if ::Rails.application.config.assets.digest
+
+ # Other gems may want to add hooks to run after the 'assets:precompile:***' tasks.
+ # Since we aren't running separate rake tasks anymore,
+ # we need to manually invoke any extra actions.
+ %w(primary nondigest).each do |asset_type|
+ Rake::Task["assets:precompile:#{asset_type}"].actions[1..-1].each &:call
+ end
end
- task :primary => ["assets:environment", "tmp:cache:clear"] do
+ task :primary => ["assets:cache:clean"] do
internal_precompile
end
- task :nondigest => ["assets:environment", "tmp:cache:clear"] do
+ task :nondigest => ["assets:cache:clean"] do
internal_precompile(false)
end
end
@@ -81,18 +97,24 @@ namespace :assets do
end
namespace :clean do
- task :all => ["assets:environment", "tmp:cache:clear"] do
- config = Rails.application.config
- public_asset_path = File.join(Rails.public_path, config.assets.prefix)
+ task :all => ["assets:cache:clean"] do
+ config = ::Rails.application.config
+ public_asset_path = File.join(::Rails.public_path, config.assets.prefix)
rm_rf public_asset_path, :secure => true
end
end
+ namespace :cache do
+ task :clean => ['tmp:create', "assets:environment"] do
+ ::Rails.application.assets.cache.clear
+ end
+ end
+
task :environment do
- if Rails.application.config.assets.initialize_on_precompile
+ if ::Rails.application.config.assets.initialize_on_precompile
Rake::Task["environment"].invoke
else
- Rails.application.initialize!(:assets)
+ ::Rails.application.initialize!(:assets)
Sprockets::Bootstrap.new(Rails.application).run
end
end
View
8 actionpack/lib/sprockets/helpers/rails_helper.rb
@@ -10,7 +10,7 @@ def asset_paths
@asset_paths ||= begin
paths = RailsHelper::AssetPaths.new(config, controller)
paths.asset_environment = asset_environment
- paths.asset_digests = asset_digests
+ paths.digests = digests
paths.compile_assets = compile_assets?
paths.digest_assets = digest_assets?
paths
@@ -95,7 +95,7 @@ def asset_prefix
Rails.application.config.assets.prefix
end
- def asset_digests
+ def digests
Rails.application.config.assets.digests
end
@@ -115,7 +115,7 @@ def asset_environment
end
class AssetPaths < ::ActionView::AssetPaths #:nodoc:
- attr_accessor :asset_environment, :asset_prefix, :asset_digests, :compile_assets, :digest_assets
+ attr_accessor :asset_environment, :asset_prefix, :digests, :compile_assets, :digest_assets
class AssetNotPrecompiledError < StandardError; end
@@ -129,7 +129,7 @@ def asset_for(source, ext)
end
def digest_for(logical_path)
- if digest_assets && asset_digests && (digest = asset_digests[logical_path])
+ if digest_assets && digests && (digest = digests[logical_path])
return digest
end
View
15 actionpack/lib/sprockets/railtie.rb
@@ -7,6 +7,7 @@ module Sprockets
autoload :LazyCompressor, "sprockets/compressors"
autoload :NullCompressor, "sprockets/compressors"
autoload :StaticCompiler, "sprockets/static_compiler"
+ autoload :StaticNonDigestGenerator, "sprockets/static_non_digest_generator"
# TODO: Get rid of config.assets.enabled
class Railtie < ::Rails::Railtie
@@ -32,15 +33,11 @@ class Railtie < ::Rails::Railtie
end
end
- if config.assets.manifest
- path = File.join(config.assets.manifest, "manifest.yml")
- else
- path = File.join(Rails.public_path, config.assets.prefix, "manifest.yml")
- end
-
- if File.exist?(path)
- config.assets.digests = YAML.load_file(path)
- end
+ manifest_dir = config.assets.manifest || File.join(Rails.public_path, config.assets.prefix)
+ digests_manifest = File.join(manifest_dir, "manifest.yml")
+ sources_manifest = File.join(manifest_dir, "sources_manifest.yml")
+ config.assets.digests = (File.exist?(digests_manifest) && YAML.load_file(digests_manifest)) || {}
+ config.assets.source_digests = (File.exist?(sources_manifest) && YAML.load_file(sources_manifest)) || {}
ActiveSupport.on_load(:action_view) do
include ::Sprockets::Helpers::RailsHelper
View
80 actionpack/lib/sprockets/static_compiler.rb
@@ -8,27 +8,82 @@ def initialize(env, target, paths, options = {})
@env = env
@target = target
@paths = paths
- @digest = options.key?(:digest) ? options.delete(:digest) : true
- @manifest = options.key?(:manifest) ? options.delete(:manifest) : true
+ @digest = options.fetch(:digest, true)
+ @manifest = options.fetch(:manifest, true)
@manifest_path = options.delete(:manifest_path) || target
+
+ @current_source_digests = options.fetch(:source_digests, {})
+ @current_digests = options.fetch(:digests, {})
+
+ @digests = {}
+ @source_digests = {}
end
def compile
- manifest = {}
+ start_time = Time.now.to_f
+
+ # Run asset compilation
+ process_assets
+
+ # Encode all filenames & digests as UTF-8 for Ruby 1.9,
+ # otherwise YAML dumps other string encodings as !binary
+ if RUBY_VERSION.to_f >= 1.9
+ @source_digests = encode_hash_as_utf8 @source_digests
+ @digests = encode_hash_as_utf8 @digests
+ end
+
+ if @manifest
+ write_manifest(@digests, @source_digests)
+ end
+
+ # Store digests in Rails config. (Important if non-digest is run after primary)
+ config = ::Rails.application.config
+ config.assets.digests = @digests
+ config.assets.source_digests = @source_digests
+
+ elapsed_time = ((Time.now.to_f - start_time) * 1000).to_i
+ env.logger.debug "Processed #{'non-' unless @digest}digest assets in #{elapsed_time}ms"
+ end
+
+ # Compiles assets if their source digests haven't changed
+ def process_assets
env.each_logical_path(paths) do |logical_path|
- if asset = env.find_asset(logical_path)
- digest_path = write_asset(asset)
- manifest[asset.logical_path] = digest_path
- manifest[aliased_path_for(asset.logical_path)] = digest_path
+ # Fetch asset without any processing or compression,
+ # to calculate a digest of the concatenated source files
+ asset = env.find_asset(logical_path, :process => false)
+
+ @source_digests[logical_path] = asset.digest
+
+ # Recompile if digest has changed or compiled digest file is missing
+ current_digest_file = @current_digests[logical_path]
+
+ if @source_digests[logical_path] != @current_source_digests[logical_path] ||
+ !(current_digest_file && File.exists?("#{@target}/#{current_digest_file}"))
+
+ if asset = env.find_asset(logical_path)
+ digest_path = write_asset(asset)
+ @digests[asset.logical_path] = digest_path
+ @digests[aliased_path_for(asset.logical_path)] = digest_path
+ end
+ else
+ # Set asset file from manifest.yml
+ digest_path = @current_digests[logical_path]
+ @digests[logical_path] = digest_path
+ @digests[aliased_path_for(logical_path)] = digest_path
+
+ env.logger.debug "Not compiling #{logical_path}, sources digest has not changed " <<
+ "(#{@source_digests[logical_path][0...7]})"
end
end
- write_manifest(manifest) if @manifest
end
- def write_manifest(manifest)
+ def write_manifest(digests, source_digests)
FileUtils.mkdir_p(@manifest_path)
File.open("#{@manifest_path}/manifest.yml", 'wb') do |f|
- YAML.dump(manifest, f)
+ YAML.dump(digests, f)
+ end
+ File.open("#{@manifest_path}/sources_manifest.yml", 'wb') do |f|
+ YAML.dump(source_digests, f)
end
end
@@ -41,10 +96,15 @@ def write_asset(asset)
end
end
+
def path_for(asset)
@digest ? asset.digest_path : asset.logical_path
end
+ def encode_hash_as_utf8(hash)
+ Hash[*hash.map {|k,v| [k.encode("UTF-8"), v.encode("UTF-8")] }.flatten]
+ end
+
def aliased_path_for(logical_path)
if File.basename(logical_path).start_with?('index')
logical_path.sub(/\/index([^\/]+)$/, '\1')
View
100 actionpack/lib/sprockets/static_non_digest_generator.rb
@@ -0,0 +1,100 @@
+require 'fileutils'
+
+module Sprockets
+ class StaticNonDigestGenerator
+
+ DIGEST_REGEX = /-([0-9a-f]{32})\./
+
+ attr_accessor :env, :target, :paths
+
+ def initialize(env, target, paths, options = {})
+ @env = env
+ @target = target
+ @paths = paths
+ @digests = options.fetch(:digests, {})
+
+ # Parse digests from digests hash
+ @asset_digests = Hash[*@digests.map {|file, digest_file|
+ [file, digest_file[DIGEST_REGEX, 1]]
+ }.flatten]
+ end
+
+
+ # Generate non-digest assets by making a copy of the digest asset,
+ # with digests stripped from js and css. The new files are also gzipped.
+ # Other assets are copied verbatim.
+ def generate
+ start_time = Time.now.to_f
+
+ env.each_logical_path(paths) do |logical_path|
+ digest_path = @digests[logical_path]
+ abs_digest_path = "#{@target}/#{digest_path}"
+ abs_logical_path = "#{@target}/#{logical_path}"
+
+ mtime = File.mtime(abs_digest_path)
+
+ # Remove known digests from css & js
+ if abs_digest_path.match(/\.(?:js|css)$/)
+ asset_body = File.read(abs_digest_path)
+
+ # Find all hashes in the asset body with a leading '-'
+ asset_body.gsub!(DIGEST_REGEX) do |match|
+ # Only remove if known digest
+ $1.in?(@asset_digests.values) ? '.' : match
+ end
+
+ # Write non-digest file
+ File.open abs_logical_path, 'w' do |f|
+ f.write asset_body
+ end
@jeremy
Ruby on Rails member
jeremy added a note Oct 7, 2012

This may change the file's encoding

Have just tested with some Japanese characters, and didn't see any problems with encoding.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ # Set modification and access times
+ File.utime(File.atime(abs_digest_path), mtime, abs_logical_path)
+
+ # Also write gzipped asset
+ File.open("#{abs_logical_path}.gz", 'wb') do |f|
+ gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION)
+ gz.mtime = mtime.to_i
+ gz.write asset_body
+ gz.close
+ end
+
+ env.logger.debug "Stripped digests, copied to #{logical_path}, and created gzipped asset"
+
+ else
+ # Otherwise, treat file as binary and copy it.
+ # Ignore paths that have no digests, such as READMEs
+ unless abs_digest_path == abs_logical_path
+ FileUtils.cp_r abs_digest_path, abs_logical_path, :remove_destination => true
+ env.logger.debug "Copied binary asset to #{logical_path}"
+
+ # Copy gzipped asset if exists
+ if File.exist? "#{abs_digest_path}.gz"
+ FileUtils.cp_r "#{abs_digest_path}.gz", "#{abs_logical_path}.gz", :remove_destination => true
+ env.logger.debug "Copied gzipped asset to #{logical_path}.gz"
+ end
+ end
+ end
+ end
+
+
+ elapsed_time = ((Time.now.to_f - start_time) * 1000).to_i
+ env.logger.debug "Generated non-digest assets in #{elapsed_time}ms"
+ end
+
+ private
+
+ def compile_path?(logical_path)
+ paths.each do |path|
+ case path
+ when Regexp
+ return true if path.match(logical_path)
+ when Proc
+ return true if path.call(logical_path)
+ else
+ return true if File.fnmatch(path.to_s, logical_path)
+ end
+ end
+ false
+ end
+ end
+end