From 37439f7c6e12b77d3c4541dfaa4286213e77bb8d Mon Sep 17 00:00:00 2001 From: Nathan Kleyn Date: Fri, 23 Jan 2015 11:53:47 +0000 Subject: [PATCH] Change tracking, env class, removed VCS code. Bump to v0.2. --- .gitignore | 1 + CHANGELOG | 7 ++ Gemfile.lock | 2 +- lib/shanty.rb | 5 +- lib/shanty/discoverer.rb | 8 +- lib/shanty/discoverers/rubygem.rb | 2 +- lib/shanty/discoverers/shantyfile.rb | 2 +- lib/shanty/env.rb | 51 ++++++++++++ lib/shanty/mutator.rb | 11 ++- lib/shanty/mutators/bundler.rb | 2 +- lib/shanty/mutators/changed.rb | 78 +++++++++++++++++++ lib/shanty/mutators/git.rb | 22 ------ lib/shanty/task_env.rb | 51 ++---------- lib/shanty/vcs_range.rb | 31 -------- lib/shanty/vcs_ranges/local_git.rb | 20 ----- shanty.gemspec | 2 +- .../lib/shanty/shanty_file_discoverer_spec.rb | 8 +- 17 files changed, 169 insertions(+), 134 deletions(-) create mode 100644 CHANGELOG create mode 100644 lib/shanty/env.rb create mode 100644 lib/shanty/mutators/changed.rb delete mode 100644 lib/shanty/mutators/git.rb delete mode 100644 lib/shanty/vcs_range.rb delete mode 100644 lib/shanty/vcs_ranges/local_git.rb diff --git a/.gitignore b/.gitignore index b06bc6a..b3823fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_Store *.gem coverage +.shanty diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..3d7412e --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,7 @@ +# Changelog + +## v0.2.0 (22nd January, 2015) + +* **Change tracking:** Added working changed flags to projects to allow CLI tasks to run just on projects that are different since last time. Note that this is a work in progress, as there are lots of things to still work on here (for example, it's currently implemented as a mutator, and it saves the change index even if the CLI command doesn't end up using it or fails to run). +* **Env class:** Added Env class to allow passing around of config, paths and other stuff separate to the TaskEnv (which contains the graph). An env instance is available to all mutators and discovers. TaskEnv is just a decorator of Env, so you can continue to call the methods you used to on it where you only have TaskEnv available (eg. in a task)! +* **Removed VCS/Git Code:** Now that we have change support that doesn't need a VCS, all of the VCS and Git stuff has been removed. This was all internal for now anyway, so shouldn't effect anybody. However, this does mean you can now use Shanty in any directory, rather than it having to be a VCS repository of some sort. diff --git a/Gemfile.lock b/Gemfile.lock index 6465c1e..a1a0b07 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - shanty (0.1.0) + shanty (0.2.0) algorithms (~> 0.6) commander (~> 4.2) deep_merge (~> 1.0) diff --git a/lib/shanty.rb b/lib/shanty.rb index b627d58..8ca5061 100644 --- a/lib/shanty.rb +++ b/lib/shanty.rb @@ -5,9 +5,10 @@ require 'shanty/cli' require 'shanty/discoverers/shantyfile' require 'shanty/discoverers/rubygem' +require 'shanty/env' require 'shanty/graph' require 'shanty/mutators/bundler' -require 'shanty/mutators/git' +require 'shanty/mutators/changed' require 'shanty/plugins/rspec' require 'shanty/plugins/rubocop' require 'shanty/task_env' @@ -23,7 +24,7 @@ class Shanty def start! setup_i18n - Cli.new(TaskEnv.new).run + Cli.new(TaskEnv.new(Env.new)).run end private diff --git a/lib/shanty/discoverer.rb b/lib/shanty/discoverer.rb index 061d2c0..de8136d 100644 --- a/lib/shanty/discoverer.rb +++ b/lib/shanty/discoverer.rb @@ -10,6 +10,12 @@ class << self attr_reader :discoverers end + attr_reader :env + + def initialize(env) + @env = env + end + def self.inherited(discoverer) Util.logger.debug("Detected project discoverer #{discoverer}") @discoverers ||= [] @@ -18,7 +24,7 @@ def self.inherited(discoverer) def discover_all self.class.discoverers.flat_map do |discoverer| - discoverer.new.discover + discoverer.new(env).discover end end diff --git a/lib/shanty/discoverers/rubygem.rb b/lib/shanty/discoverers/rubygem.rb index a26ac78..505f893 100644 --- a/lib/shanty/discoverers/rubygem.rb +++ b/lib/shanty/discoverers/rubygem.rb @@ -7,7 +7,7 @@ module Shanty # a directory class RubygemDiscoverer < Discoverer def discover - Dir['**/*.gemspec'].map do |path| + Dir[File.join(env.root, '**', '*.gemspec')].map do |path| create_project_template(File.absolute_path(File.dirname(path))) do |project_template| project_template.type = RubygemProject end diff --git a/lib/shanty/discoverers/shantyfile.rb b/lib/shanty/discoverers/shantyfile.rb index 7226fe4..737e742 100644 --- a/lib/shanty/discoverers/shantyfile.rb +++ b/lib/shanty/discoverers/shantyfile.rb @@ -12,7 +12,7 @@ module Shanty # customisation. class ShantyfileDiscoverer < Discoverer def discover - Dir['**/Shantyfile'].map do |path| + Dir[File.join(env.root, '**', 'Shantyfile')].map do |path| create_project_template(File.absolute_path(File.dirname(path))) do |project_template| project_template.priority = -1 end diff --git a/lib/shanty/env.rb b/lib/shanty/env.rb new file mode 100644 index 0000000..b3586a7 --- /dev/null +++ b/lib/shanty/env.rb @@ -0,0 +1,51 @@ +require 'deep_merge' +require 'yaml' + +module Shanty + # + class Env + CONFIG_FILE = '.shanty.yml' + DEFAULT_CONFIG = {} + + def initialize + Dir.chdir(root) do + (config['require'] || {}).each do |requirement| + requirement = "#{requirement}/**/*.rb" unless requirement.include?('.rb') + Dir[requirement].each { |f| require(File.join(root, f)) } + end + end + end + + def environment + @environment = ENV['SHANTY_ENV'] || 'local' + end + + def root + @root ||= find_root + end + + private + + def config + return @config unless @config.nil? + + file_config = YAML.load_file("#{root}/#{CONFIG_FILE}") || {} + @config = DEFAULT_CONFIG.deep_merge!(file_config[environment]) + end + + def find_root + if root_dir.nil? + fail "Could not find a #{CONFIG_FILE} file in this or any parent directories. \ + Please run `shanty init` in the directory you want to be the root of your project structure." + end + + root_dir + end + + def root_dir + Pathname.new(Dir.pwd).ascend do |d| + return d if d.join(CONFIG_FILE).exist? + end + end + end +end diff --git a/lib/shanty/mutator.rb b/lib/shanty/mutator.rb index 078852c..73a7f8b 100644 --- a/lib/shanty/mutator.rb +++ b/lib/shanty/mutator.rb @@ -8,15 +8,22 @@ class << self attr_reader :mutators end + attr_reader :env, :graph + + def initialize(env, graph) + @env = env + @graph = graph + end + def self.inherited(mutator) Util.logger.debug("Detected mutator #{mutator}") @mutators ||= [] @mutators << mutator end - def apply_mutations(graph) + def apply_mutations self.class.mutators.each do |mutator| - mutator.new.mutate(graph) + mutator.new(@env, @graph).mutate end end end diff --git a/lib/shanty/mutators/bundler.rb b/lib/shanty/mutators/bundler.rb index f385ee6..f37718e 100644 --- a/lib/shanty/mutators/bundler.rb +++ b/lib/shanty/mutators/bundler.rb @@ -4,7 +4,7 @@ module Shanty # Bundler mutator class BundlerMutator < Mutator - def mutate(graph) + def mutate graph.each do |project| BundlerPlugin.add_to_project(project) if File.exist?(File.join(project.path, 'Gemfile')) end diff --git a/lib/shanty/mutators/changed.rb b/lib/shanty/mutators/changed.rb new file mode 100644 index 0000000..c2f3b16 --- /dev/null +++ b/lib/shanty/mutators/changed.rb @@ -0,0 +1,78 @@ +require 'find' +require 'fileutils' +require 'set' +require 'shanty/mutator' + +module Shanty + # Mutates the graph to mark projects which have changed since the last time they were built. + class ChangedMutator < Mutator + UNIT_SEPARATOR = "\u001F" + + def mutate + FileUtils.mkdir(shanty_dir) unless File.directory?(shanty_dir) + + self.cached_index = all_index_files.each_with_object({}) do |path, acc| + # Check if the file has changed between + next if unchanged_in_index?(path) + # Otherwise, it was modified, deleted or added, so update the index if the file still exists. + acc[path] = current_index[path] if current_index.include?(path) + mark_project_as_changed(path) + end + end + + private + + def shanty_dir + File.join(env.root, '.shanty') + end + + def index_file + File.join(shanty_dir, 'index') + end + + def all_index_files + Set.new(cached_index.keys + current_index.keys).to_a + end + + def cached_index + return @cached_index unless @cached_index.nil? + return (@cached_index = {}) unless File.exist?(index_file) + + @cached_index = File.open(index_file).each_line.each_with_object({}) do |line, acc| + path, *attrs = line.split(UNIT_SEPARATOR) + acc[path] = attrs + end + end + + def cached_index=(new_index) + File.open(index_file, 'w+') do |f| + new_index.each do |path, attrs| + f.puts(attrs.unshift(path).join(UNIT_SEPARATOR)) + end + end + end + + def unchanged_in_index?(path) + cached = cached_index[path] + current = current_index[path] + !cached.nil? && !current.nil? && current == cached + end + + def current_index + @current_index ||= Find.find(env.root).each_with_object({}) do |path, acc| + # FIXME: Pass in list of excludes and match as follows: + # next Find.prune if path =~ /(build|.git|.gradle)/ + next unless File.exist?(path) + s = File.stat(path) + next if s.directory? + + acc[path] = [s.mtime.to_i, s.size] + end + end + + def mark_project_as_changed(path) + project = graph.owner_of_file(File.join(env.root, path)) + project.changed = true unless project.nil? + end + end +end diff --git a/lib/shanty/mutators/git.rb b/lib/shanty/mutators/git.rb deleted file mode 100644 index 8bb95e8..0000000 --- a/lib/shanty/mutators/git.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'shanty/mutator' -require 'shanty/vcs_ranges/local_git' - -module Shanty - # Git VCS mutator - class GitMutator < Mutator - def initialize - @vcs_range = VCSRange.new - end - - def mutate(graph) - git_root = `git rev-parse --show-toplevel`.strip - diff_files = `git diff --name-only #{@vcs_range.from_commit} #{@vcs_range.to_commit} 2>/dev/null`.split("\n") - diff_files.each do |path| - next if path.nil? - path = File.join(git_root, path) - project = graph.owner_of_file(path) - project.changed = true unless project.nil? - end - end - end -end diff --git a/lib/shanty/task_env.rb b/lib/shanty/task_env.rb index 8dc1933..1f9253e 100644 --- a/lib/shanty/task_env.rb +++ b/lib/shanty/task_env.rb @@ -1,64 +1,23 @@ -require 'deep_merge' -require 'yaml' +require 'delegate' module Shanty # - class TaskEnv - CONFIG_FILE = '.shanty.yml' - DEFAULT_CONFIG = {} - - def initialize - Dir.chdir(root) do - (config['require'] || {}).each do |requirement| - requirement = "#{requirement}/**/*.rb" unless requirement.include?('.rb') - Dir[requirement].each { |f| require(File.join(root, f)) } - end - end - end + class TaskEnv < SimpleDelegator + alias_method :env, :__getobj__ def graph @graph ||= construct_project_graph end - def environment - @environment = ENV['SHANTY_ENV'] || 'local' - end - - def root - @root ||= find_root - end - private - def config - return @config unless @config.nil? - - file_config = YAML.load_file("#{root}/#{CONFIG_FILE}") || {} - @config = DEFAULT_CONFIG.deep_merge!(file_config[environment]) - end - - def find_root - if root_dir.nil? - fail "Could not find a #{CONFIG_FILE} file in this or any parent directories. \ - Please run `shanty init` in the directory you want to be the root of your project structure." - end - - root_dir - end - - def root_dir - Pathname.new(Dir.pwd).ascend do |d| - return d if d.join(CONFIG_FILE).exist? - end - end - def construct_project_graph project_templates = Dir.chdir(root) do - Discoverer.new.discover_all.sort_by(&:priority).reverse.uniq(&:path) + Discoverer.new(env).discover_all.sort_by(&:priority).reverse.uniq(&:path) end Graph.new(project_templates).tap do |graph| - Mutator.new.apply_mutations(graph) + Mutator.new(env, graph).apply_mutations end end end diff --git a/lib/shanty/vcs_range.rb b/lib/shanty/vcs_range.rb deleted file mode 100644 index bdaa539..0000000 --- a/lib/shanty/vcs_range.rb +++ /dev/null @@ -1,31 +0,0 @@ -require 'shanty/util' - -module Shanty - # Public: Enables discovery of rangeerent types of CI provider - # utilises inherited class method to find all implementing - # classes - class VCSRange - class << self - attr_reader :vcs_range - end - - def self.inherited(vcs_range) - Util.logger.debug("Detected VCS range #{vcs_range}") - @vcs_range = vcs_range - end - - def from_commit - vcs_range.from_commit - end - - def to_commit - vcs_range.to_commit - end - - private - - def vcs_range - @vcs_range ||= self.class.vcs_range.new - end - end -end diff --git a/lib/shanty/vcs_ranges/local_git.rb b/lib/shanty/vcs_ranges/local_git.rb deleted file mode 100644 index a520db9..0000000 --- a/lib/shanty/vcs_ranges/local_git.rb +++ /dev/null @@ -1,20 +0,0 @@ -require 'shanty/vcs_range' - -module Shanty - # Use range of local vs remote changes - class LocalGit < VCSRange - def from_commit - "origin/#{branch}" - end - - def to_commit - 'HEAD' - end - - private - - def branch - `git rev-parse --abbrev-ref HEAD`.strip - end - end -end diff --git a/shanty.gemspec b/shanty.gemspec index 2bff836..bfdef69 100644 --- a/shanty.gemspec +++ b/shanty.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |gem| gem.name = 'shanty' - gem.version = '0.1.0' + gem.version = '0.2.0' gem.homepage = 'https://github.com/shantytown/shanty' gem.license = 'MIT' diff --git a/spec/lib/shanty/shanty_file_discoverer_spec.rb b/spec/lib/shanty/shanty_file_discoverer_spec.rb index 4babc58..5ecc2fd 100644 --- a/spec/lib/shanty/shanty_file_discoverer_spec.rb +++ b/spec/lib/shanty/shanty_file_discoverer_spec.rb @@ -1,5 +1,6 @@ require 'spec_helper' +require 'shanty/env' require 'shanty/discoverers/shantyfile' # All classes referenced belong to the shanty project @@ -17,11 +18,8 @@ module Shanty File.join(Dir.pwd, 'examples', project2, project3) ] end - let(:project_templates) do - Dir.chdir('examples') do - ShantyfileDiscoverer.new.discover - end - end + let(:env) { Dir.chdir('examples') { Env.new } } + let(:project_templates) { ShantyfileDiscoverer.new(env).discover } describe '#discovered' do it 'finds example projects' do