Skip to content
Browse files

Merge pull request #117 from twinturbo/performance

Performance Fixes: from the very small to very large
  • Loading branch information...
2 parents 082e4c2 + c71efaa commit 234f8a33a31285331206f46d1ceab7929b049d0b @wycats wycats committed Oct 31, 2012
View
29 lib/rake-pipeline.rb
@@ -333,6 +333,7 @@ def invoke
def setup
setup_filters
generate_rake_tasks
+ record_input_files
end
# Set up the filters. This will loop through all of the filters for
@@ -393,17 +394,27 @@ def finalize
# A unique fingerprint. It's used to generate unique temporary
# directory names. It must be unique to the pipeline. It must be
# the same across processes.
+ #
+ # @return [String]
+ # @api private
def fingerprint
- files = eligible_input_files + output_files
-
- Digest::MD5.hexdigest(files.to_s)[0..7]
+ if project
+ project.pipelines.index self
+ else
+ 1
+ end
end
# the Manifest used in this pipeline
def manifest
project.manifest
end
+ # the Manifest used in this pipeline
+ def last_manifest
+ project.last_manifest
+ end
+
protected
# Generate a new temporary directory name.
#
@@ -437,5 +448,17 @@ def assert_input_provided
end
end
+ # This is needed to ensure that every file procesed in the pipeline
+ # has an entry in the manifest. This is used to compare input files
+ # for the global dirty check
+ def record_input_files
+ input_files.each do |file|
+ full_path = file.fullpath
+
+ if File.exists?(full_path) && !manifest[full_path]
+ manifest[full_path] ||= ManifestEntry.new({}, File.mtime(full_path).to_i)
+ end
+ end
+ end
end
end
View
26 lib/rake-pipeline/dynamic_file_task.rb
@@ -26,7 +26,7 @@ def to_s
end
end
- attr_accessor :manifest
+ attr_accessor :manifest, :last_manifest
# @return [Boolean] true if the task has a block to invoke
# for dynamic dependencies, false otherwise.
@@ -49,6 +49,10 @@ def manifest_entry=(new_entry)
manifest[name] = new_entry
end
+ def last_manifest_entry
+ last_manifest[name]
+ end
+
# Invoke this task. This method only checks to see if there
# is a manifest then delegates to super
def invoke(*args)
@@ -67,12 +71,12 @@ def needed?
return true if prerequisites_needed?
# if we have no manifest, this file task is needed
- return true unless manifest_entry
+ return true unless last_manifest_entry
# If any of this task's dynamic dependencies have changed,
# this file task is needed
- manifest_entry.deps.each do |dep, time|
- return true if File.mtime(dep) > time || time > timestamp
+ last_manifest_entry.deps.each do |dep, time|
+ return true if File.mtime(dep).to_i > time
end
# Otherwise, it's not needed
@@ -138,7 +142,7 @@ def invoke_prerequisites(task_args, invocation_chain)
entry = Rake::Pipeline::ManifestEntry.new
dynamics.each do |dynamic|
- entry.deps.merge!(dynamic => mtime_or_now(dynamic))
+ entry.deps.merge!(dynamic => mtime_or_now(dynamic).to_i)
end
self.manifest_entry = entry
@@ -149,7 +153,7 @@ def invoke_prerequisites(task_args, invocation_chain)
def invoke_with_call_chain(*)
super
- manifest_entry.mtime = mtime_or_now(name)
+ manifest_entry.mtime = mtime_or_now(name).to_i
end
private
@@ -167,22 +171,22 @@ def dynamic_prerequisites_from_manifest
# Try to avoid invoking the dynamic block if this file
# is not needed. If so, we may have all the information
# we need in the manifest file.
- if !needed? && manifest_entry
- mtime = manifest_entry.mtime
+ if !needed? && last_manifest_entry
+ mtime = last_manifest_entry.mtime
end
# If the output file of this task still exists and
# it hasn't been updated, we can simply return the
# list of dependencies in the manifest, which
# come from the return value of the dynamic block
# in a previous run.
- if File.exist?(name) && mtime == File.mtime(name)
- return manifest_entry.deps.map { |k,v| k }
+ if File.exist?(name) && mtime == File.mtime(name).to_i
+ return last_manifest_entry.deps.map { |k,v| k }
end
end
def prerequisites_needed?
- prerequisite_tasks.any? { |n| application[n, @scope].needed? }
+ prerequisite_tasks.any? { |n| n.needed? }
end
end
end
View
8 lib/rake-pipeline/filter.rb
@@ -72,7 +72,7 @@ class Filter
# this filter.
attr_accessor :pipeline
- attr_writer :manifest
+ attr_writer :manifest, :last_manifest
attr_writer :file_wrapper_class
@@ -91,6 +91,10 @@ def manifest
@manifest || pipeline.manifest
end
+ def last_manifest
+ @last_manifest || pipeline.last_manifest
+ end
+
# Invoke this method in a subclass of Filter to declare that
# it expects to work with BINARY data, and that data that is
# not valid UTF-8 should be allowed.
@@ -203,6 +207,7 @@ def generate_rake_tasks
@rake_tasks = outputs.map do |output, inputs|
additional_paths = []
inputs.each do |input|
+
create_file_task(input.fullpath).dynamic do
additional_paths += additional_dependencies(input)
end
@@ -224,6 +229,7 @@ def encoding
def create_file_task(output, deps=[], &block)
task = rake_application.define_task(Rake::Pipeline::DynamicFileTask, output => deps, &block)
task.manifest = manifest
+ task.last_manifest = last_manifest
task
end
View
19 lib/rake-pipeline/manifest.rb
@@ -58,6 +58,25 @@ def [](key)
def []=(key, value)
@entries[key] = value
end
+
+ def empty?
+ entries.empty?
+ end
+
+ def files
+ entries.inject({}) do |hash, pair|
+ file = pair.first
+ entry = pair.last
+
+ hash.merge!(file => entry.mtime)
+
+ entry.deps.each_pair do |name, time|
+ hash.merge!(name => time)
+ end
+
+ hash
+ end
+ end
end
end
end
View
4 lib/rake-pipeline/manifest_entry.rb
@@ -6,10 +6,10 @@ class ManifestEntry
def self.from_hash(hash)
entry = new
- entry.mtime = DateTime.parse(hash["mtime"]).to_time
+ entry.mtime = hash["mtime"]
hash["deps"].each do |dep, time_string|
- entry.deps[dep] = DateTime.parse(time_string).to_time
+ entry.deps[dep] = time_string
end
entry
View
7 lib/rake-pipeline/middleware.rb
@@ -17,11 +17,10 @@ class Middleware
attr_accessor :project
# @param [#call] app a Rack application
- # @param [String|Rake::Pipeline] pipeline either a path to an
- # Assetfile to use to build a pipeline, or an existing pipeline.
- def initialize(app, pipeline)
+ # @param [Rake::Pipeline::Project] an existing project
+ def initialize(app, project)
@app = app
- @project = Rake::Pipeline::Project.new(pipeline)
+ @project = project
end
# Automatically compiles your assets if required and
View
75 lib/rake-pipeline/project.rb
@@ -110,16 +110,15 @@ def build(&block)
# @see Rake::Pipeline#invoke
def invoke
@invoke_mutex.synchronize do
- if assetfile_path
- source = File.read(assetfile_path)
- if digest(source) != assetfile_digest
- rebuild_from_assetfile(assetfile_path, source)
- end
- end
+ last_manifest.read_manifest
+
+ if dirty?
+ rebuild_from_assetfile(assetfile_path) if assetfile_dirty?
+
+ pipelines.each(&:invoke)
- manifest.read_manifest
- pipelines.each(&:invoke)
- manifest.write_manifest
+ manifest.write_manifest
+ end
end
end
@@ -215,6 +214,12 @@ def manifest
@manifest ||= Rake::Pipeline::Manifest.new(manifest_path)
end
+ # @return [Manifest] the manifest to write dependency information
+ # to
+ def last_manifest
+ @last_manifest ||= Rake::Pipeline::Manifest.new(manifest_path)
+ end
+
# @return [String] the path to the dynamic dependency manifest
def manifest_path
File.join(digested_tmpdir, "manifest.json")
@@ -258,6 +263,58 @@ def setup_pipelines
def digest(str)
Digest::SHA1.hexdigest(str)
end
+
+ def dirty?
+ assetfile_dirty? || files_dirty?
+ end
+
+ def assetfile_dirty?
+ if assetfile_path
+ source = File.read(assetfile_path)
+ digest(source) != assetfile_digest
+ else
+ false
+ end
+ end
+
+ # Returns true if any of these conditions are met:
+ # The pipeline hasn't been invoked yet
+ # The input files have been modified
+ # Any of the input files have been deleted
+ # There are new input files
+ def files_dirty?
+ return true if manifest.empty?
+
+ previous_files = manifest.files
+
+ input_files.each do |input_file|
+ if !File.exists? input_file
+ return true # existing input file has been deleted
+ elsif !previous_files[input_file]
+ return true # there is a new file in the pipeline
+ elsif File.mtime(input_file).to_i != previous_files[input_file]
+ return true # existing file has been changed
+ end
+ end
+
+ false
+ end
+
+ def input_files
+ static_input_files = pipelines.collect do |p|
+ p.input_files.reject { |file| file.in_directory? tmpdir }.map(&:fullpath)
+ end.flatten
+
+ dynamic_input_files = static_input_files.collect do |file|
+ if manifest[file]
+ manifest[file].deps.keys
+ else
+ nil
+ end
+ end.flatten.compact
+
+ static_input_files + dynamic_input_files
+ end
end
end
end
View
2 lib/rake-pipeline/server.rb
@@ -6,7 +6,7 @@ class Pipeline
class Server < Rack::Server
def app
not_found = proc { [404, { "Content-Type" => "text/plain" }, ["not found"]] }
- config = "Assetfile"
+ project = Rake::Pipline::Project.new "Assetfile"
Middleware.new(not_found, config)
end
View
1 spec/concat_filter_spec.rb
@@ -12,6 +12,7 @@
it "generates output" do
filter = Rake::Pipeline::ConcatFilter.new { "application.js" }
filter.manifest = MemoryManifest.new
+ filter.last_manifest = MemoryManifest.new
filter.file_wrapper_class = MemoryFileWrapper
filter.output_root = "/path/to/output"
filter.input_files = input_files
View
15 spec/dynamic_file_task_spec.rb
@@ -20,11 +20,6 @@ def define_task(deps, klass=Rake::Pipeline::DynamicFileTask, &task_proc)
let(:task) { define_task('output') }
- before do
- # Make sure date conversions happen in UTC, not local time
- ENV['TZ'] = 'UTC'
- end
-
after do
# Clean out all defined tasks after each test runs
Rake.application = Rake::Application.new
@@ -57,13 +52,13 @@ def define_task(deps, klass=Rake::Pipeline::DynamicFileTask, &task_proc)
it "adds dynamic dependencies to its manifest entry" do
dynamic_task.invoke
dynamic_task.manifest_entry.deps.should == {
- 'dynamic' => File.mtime('dynamic')
+ 'dynamic' => File.mtime('dynamic').to_i
}
end
it "adds the current task's mtime to its manifest entry" do
dynamic_task.invoke
- dynamic_task.manifest_entry.mtime.should == File.mtime('output')
+ dynamic_task.manifest_entry.mtime.should == File.mtime('output').to_i
end
it "raises an error when there is no manifest" do
@@ -109,13 +104,13 @@ def make_file(name, mtime=nil)
manifest_entry = Rake::Pipeline::ManifestEntry.from_hash({
"deps" => {
- "blinky" => "2000-01-01 00:00:00 +0000"
+ "blinky" => time.to_i
},
- "mtime" => "2000-01-01 00:00:00 +0000"
+ "mtime" => time.to_i
})
task.dynamic { %w[] }
- task.stub(:manifest_entry) { manifest_entry }
+ task.stub(:last_manifest_entry) { manifest_entry }
task.should_not_receive(:invoke_dynamic_block)
task.dynamic_prerequisites.should == %w[blinky]
end
View
4 spec/filter_spec.rb
@@ -1,4 +1,6 @@
describe "Rake::Pipeline::Filter" do
+ MemoryManifest ||= Rake::Pipeline::SpecHelpers::MemoryManifest
+
after do
# Clean out all defined tasks after each test runs
Rake.application = Rake::Application.new
@@ -287,6 +289,7 @@ def write_input_file(filename, contents='', root=input_root)
def invoke_filter(filter)
mkdir_p output_root
filter.pipeline = pipeline
+ filter.last_manifest = MemoryManifest.new
filter.output_root = output_root
filter.input_files = input_files
tasks = filter.generate_rake_tasks
@@ -318,6 +321,7 @@ def invoke_filter(filter)
it "rebuilds the main file when a dynamic dependency changes" do
invoke_filter(filter)
sleep 1
+ $LATCH = true
File.open(header.fullpath, 'w') { |f| f.puts 'PEANUTS' }
filter.rake_tasks.each { |t| t.recursively_reenable(Rake.application) }
invoke_filter(filter)
View
3 spec/gsub_filter_spec.rb
@@ -16,6 +16,7 @@
filter.file_wrapper_class = MemoryFileWrapper
filter.manifest = MemoryManifest.new
+ filter.last_manifest = MemoryManifest.new
filter.output_root = "/path/to/output"
filter.input_files = input_files
@@ -38,6 +39,7 @@
filter.rake_application = rake_application
filter.file_wrapper_class = MemoryFileWrapper
filter.manifest = MemoryManifest.new
+ filter.last_manifest = MemoryManifest.new
filter.output_root = "/path/to/output"
filter.input_files = input_files
@@ -60,6 +62,7 @@
filter.rake_application = rake_application
filter.file_wrapper_class = MemoryFileWrapper
filter.manifest = MemoryManifest.new
+ filter.last_manifest = MemoryManifest.new
filter.output_root = "/path/to/output"
filter.input_files = input_files
View
23 spec/manifest_entry_spec.rb
@@ -1,18 +1,13 @@
require 'spec_helper'
describe Rake::Pipeline::ManifestEntry do
- before do
- # ensure all our date/time conversions happen in UTC.
- ENV['TZ'] = 'UTC'
- end
-
let(:hash) do
{
"deps" => {
- "file.h" => "2001-01-01 00:00:00 +0000",
- "other.h" => "2001-01-01 00:00:00 +0000"
+ "file.h" => 100,
+ "other.h" => 100
},
- "mtime" => "2000-01-01 00:00:00 +0000"
+ "mtime" => 200
}
end
@@ -25,13 +20,13 @@
subject.should be_kind_of described_class
end
- it "parses mtime value into a Time" do
- subject.mtime.should == Time.utc(2000)
+ it "should leave mtimes have integers" do
+ subject.mtime.should == 200
end
it "parses each time from the deps value into a Time" do
subject.deps.each do |file, mtime|
- mtime.should == Time.utc(2001)
+ mtime.should == 100
end
end
end
@@ -40,10 +35,10 @@
it "returns a hash representation of the entry for converting to json" do
subject.as_json.should == {
:deps => {
- "file.h" => Time.utc(2001),
- "other.h" => Time.utc(2001),
+ "file.h" => 100,
+ "other.h" => 100
},
- :mtime => Time.utc(2000)
+ :mtime => 200
}
end
end
View
4 spec/middleware_spec.rb
@@ -68,8 +68,10 @@
assetfile_path = File.join(tmp, "Assetfile")
File.open(assetfile_path, "w") { |file| file.write(assetfile_source) }
+ project = Rake::Pipeline::Project.new assetfile_path
+
app = lambda { |env| [404, {}, ['not found']] }
- middleware = Rake::Pipeline::Middleware.new(app, assetfile_path)
+ middleware = Rake::Pipeline::Middleware.new(app, project)
end
describe "dynamic requests" do
View
1 spec/ordering_concat_filter_spec.rb
@@ -23,6 +23,7 @@
def make_filter(ordering)
filter = Rake::Pipeline::OrderingConcatFilter.new(ordering, "all.txt")
filter.manifest = MemoryManifest.new
+ filter.last_manifest = MemoryManifest.new
filter.file_wrapper_class = MemoryFileWrapper
filter.input_files = input_files
filter.output_root = "/path/to/output"
View
25 spec/pipeline_spec.rb
@@ -79,31 +79,6 @@
end
end
- describe "#fingerprint" do
- include Rake::Pipeline::SpecHelpers::InputHelpers
-
- it "should change when the input files changes" do
- pipeline.add_filter ConcatFilter.new("vendor.js")
- pipeline.input_files = [input_file("jquery.js")]
-
- lambda {
- pipeline.input_files = [input_file("ember.js")]
- }.should change(pipeline, :fingerprint)
- end
-
- it "should change when the output files change" do
- pipeline.output_root = "public"
-
- pipeline.add_filter ConcatFilter.new("vendor.js")
- pipeline.input_files = [input_file("jquery.js")]
-
- lambda {
- pipeline.add_filter ConcatFilter.new("application.js")
- pipeline.setup
- }.should change(pipeline, :fingerprint)
- end
- end
-
describe "adding filters" do
let(:filter) { ConcatFilter.new }
View
18 spec/project_spec.rb
@@ -104,7 +104,7 @@ def modify_assetfile
file.write(MODIFIED_ASSETFILE_SOURCE)
end
end
-
+
it "creates output files" do
output_files.each { |file| file.should_not exist }
project.invoke
@@ -117,11 +117,11 @@ def modify_assetfile
end
it "updates the manifest" do
- project.manifest.should_receive(:read_manifest)
+ project.last_manifest.should_receive(:read_manifest)
project.manifest.should_receive(:write_manifest)
project.invoke
end
-
+
it "rebuilds its pipeline when the Assetfile changes" do
project.invoke
original_pipeline = project.pipelines.last
@@ -146,18 +146,6 @@ def modify_assetfile
end
end
- describe "#invoke" do
- context "if the Assetfile contents have changed" do
-
- end
-
- it "updates the manifest" do
- project.manifest.should_receive(:read_manifest)
- project.manifest.should_receive(:write_manifest)
- project.invoke
- end
- end
-
describe "#cleanup_tmpdir" do
it "cleans old rake-pipeline-* dirs out of the pipeline's tmp dir" do
File.exist?(old_tmpdir).should be_true
View
9 spec/rake_acceptance_spec.rb
@@ -107,12 +107,14 @@ def copy_files
it "can successfully apply filters" do
concat = concat_filter.new
concat.manifest = memory_manifest.new
+ concat.last_manifest = memory_manifest.new
concat.input_files = INPUTS.keys.select { |key| key =~ /javascript/ }.map { |file| input_wrapper(file) }
concat.output_root = File.join(tmp, "temporary", "concat_filter")
concat.output_name_generator = proc { |input| "javascripts/application.js" }
strip_asserts = strip_asserts_filter.new
strip_asserts.manifest = memory_manifest.new
+ strip_asserts.last_manifest = memory_manifest.new
strip_asserts.input_files = concat.output_files
strip_asserts.output_root = File.join(tmp, "public")
strip_asserts.output_name_generator = proc { |input| input }
@@ -127,12 +129,14 @@ def copy_files
it "supports filters with multiple outputs per input" do
concat = concat_filter.new
concat.manifest = memory_manifest.new
+ concat.last_manifest = memory_manifest.new
concat.input_files = INPUTS.keys.select { |key| key =~ /javascript/ }.map { |file| input_wrapper(file) }
concat.output_root = File.join(tmp, "temporary", "concat_filter")
concat.output_name_generator = proc { |input| [ "javascripts/application.js", input.sub(/^app\//, '') ] }
strip_asserts = strip_asserts_filter.new
strip_asserts.manifest = memory_manifest.new
+ strip_asserts.last_manifest = memory_manifest.new
strip_asserts.input_files = concat.output_files
strip_asserts.output_root = File.join(tmp, "public")
strip_asserts.output_name_generator = proc { |input| input }
@@ -159,16 +163,19 @@ def copy_files
it "can be configured using the pipeline" do
pipeline = Rake::Pipeline.new
+ pipeline.project = Rake::Pipeline::Project.new
pipeline.add_input tmp, 'app/javascripts/*.js'
pipeline.output_root = File.expand_path("public")
pipeline.tmpdir = "temporary"
concat = concat_filter.new
concat.manifest = memory_manifest.new
+ concat.last_manifest = memory_manifest.new
concat.output_name_generator = proc { |input| "javascripts/application.js" }
strip_asserts = strip_asserts_filter.new
strip_asserts.manifest = memory_manifest.new
+ strip_asserts.last_manifest = memory_manifest.new
strip_asserts.output_name_generator = proc { |input| input }
pipeline.add_filters(concat, strip_asserts)
@@ -619,7 +626,7 @@ def output_should_exist(expected=EXPECTED_JS_OUTPUT)
end
end
- project.invoke_clean
+ project.invoke
output_should_exist
end

0 comments on commit 234f8a3

Please sign in to comment.
Something went wrong with that request. Please try again.