From a59f7bd791373454ff97375b5377060e5c52fe69 Mon Sep 17 00:00:00 2001 From: Michal Papis Date: Sun, 23 Nov 2014 22:30:35 +0100 Subject: [PATCH 1/8] add support for :extra_files config whitelist for extra files, update #492 --- lib/nanoc/data_sources/filesystem.rb | 2 +- lib/nanoc/data_sources/static.rb | 2 +- lib/nanoc/extra/filesystem_tools.rb | 19 +++++++++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/nanoc/data_sources/filesystem.rb b/lib/nanoc/data_sources/filesystem.rb index 014bbcfc7b..350f0aec08 100644 --- a/lib/nanoc/data_sources/filesystem.rb +++ b/lib/nanoc/data_sources/filesystem.rb @@ -179,7 +179,7 @@ def all_split_files_in(dir_name) # Returns all files in the given directory and directories below it. def all_files_in(dir_name) - Nanoc::Extra::FilesystemTools.all_files_in(dir_name) + Nanoc::Extra::FilesystemTools.all_files_in(dir_name, config[:extra_files]) end # Returns the filename for the given base filename and the extension. diff --git a/lib/nanoc/data_sources/static.rb b/lib/nanoc/data_sources/static.rb index 374826c9e2..583a8e78c4 100644 --- a/lib/nanoc/data_sources/static.rb +++ b/lib/nanoc/data_sources/static.rb @@ -54,7 +54,7 @@ def items protected def all_files_in(dir_name) - Nanoc::Extra::FilesystemTools.all_files_in(dir_name) + Nanoc::Extra::FilesystemTools.all_files_in(dir_name, config[:extra_files]) end end end diff --git a/lib/nanoc/extra/filesystem_tools.rb b/lib/nanoc/extra/filesystem_tools.rb index 59afef3c9b..61f9ef6874 100644 --- a/lib/nanoc/extra/filesystem_tools.rb +++ b/lib/nanoc/extra/filesystem_tools.rb @@ -53,8 +53,8 @@ def initialize(filename) # # @raise [UnsupportedFileTypeError] if a file of an unsupported type is # detected (something other than file, directory or link) - def all_files_in(dir_name, recursion_limit = 10) - Dir[dir_name + '/**/*'].map do |fn| + def all_files_in(dir_name, extra_files, recursion_limit = 10) + all_files_and_dirs_in(dir_name, recursion_limit).map do |fn| case File.ftype(fn) when 'link' if 0 == recursion_limit @@ -80,6 +80,21 @@ def all_files_in(dir_name, recursion_limit = 10) end module_function :all_files_in + def all_files_and_dirs_in(dir_name, extra_files) + patterns = [dir_name + '/**/*'] + case extra_files + when nil + patterns << "#{dir_name}/**/.{htaccess,htpasswd}" + when String + patterns << "#{dir_name}/#{extra_files}" + when Array + patterns += extra_files.map { |extra_file| "#{dir_name}/#{extra_file}" } + else + raise "Do not know how to handle extra_files: #{extra_files.inspect}" + end + Dir.glob(*patterns) + end + # Resolves the given symlink into an absolute path. # # @param [String] filename The filename of the symlink to resolve From 4960094859e71a6d53bae126b9f2dc3f66abd2b9 Mon Sep 17 00:00:00 2001 From: Andreas Date: Wed, 5 Nov 2014 11:23:35 +0100 Subject: [PATCH 2/8] Test support for dotfiles in data-sources .htaccess and .htpasswd were not handled by any data-source --- test/extra/test_filesystem_tools.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/extra/test_filesystem_tools.rb b/test/extra/test_filesystem_tools.rb index 7eff394713..d6187aec39 100644 --- a/test/extra/test_filesystem_tools.rb +++ b/test/extra/test_filesystem_tools.rb @@ -100,4 +100,14 @@ def test_resolve_symlink_too_many Nanoc::Extra::FilesystemTools.resolve_symlink('symlink-7') end end + + def test_dotfiles_are_valid_items + # Write sample files + FileUtils.mkdir_p('dir') + File.open('dir/.htaccess', 'w') { |io| io.write('o hai') } + + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir').sort + assert_equal ['dir/.htaccess'], actual_files + end + end From 82d8aef4ed6d997bcea84708259402f787534cf4 Mon Sep 17 00:00:00 2001 From: Michal Papis Date: Sun, 23 Nov 2014 22:49:44 +0100 Subject: [PATCH 3/8] fix code and tests --- lib/nanoc/extra/filesystem_tools.rb | 9 +++++---- test/data_sources/test_static.rb | 2 +- test/extra/test_filesystem_tools.rb | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/nanoc/extra/filesystem_tools.rb b/lib/nanoc/extra/filesystem_tools.rb index 61f9ef6874..1da117a197 100644 --- a/lib/nanoc/extra/filesystem_tools.rb +++ b/lib/nanoc/extra/filesystem_tools.rb @@ -54,7 +54,7 @@ def initialize(filename) # @raise [UnsupportedFileTypeError] if a file of an unsupported type is # detected (something other than file, directory or link) def all_files_in(dir_name, extra_files, recursion_limit = 10) - all_files_and_dirs_in(dir_name, recursion_limit).map do |fn| + all_files_and_dirs_in(dir_name, extra_files).map do |fn| case File.ftype(fn) when 'link' if 0 == recursion_limit @@ -64,7 +64,7 @@ def all_files_in(dir_name, extra_files, recursion_limit = 10) if File.file?(absolute_target) fn else - all_files_in(absolute_target, recursion_limit - 1).map do |sfn| + all_files_in(absolute_target, extra_files, recursion_limit - 1).map do |sfn| fn + sfn[absolute_target.size..-1] end end @@ -81,7 +81,7 @@ def all_files_in(dir_name, extra_files, recursion_limit = 10) module_function :all_files_in def all_files_and_dirs_in(dir_name, extra_files) - patterns = [dir_name + '/**/*'] + patterns = [ "#{dir_name}/**/*" ] case extra_files when nil patterns << "#{dir_name}/**/.{htaccess,htpasswd}" @@ -92,8 +92,9 @@ def all_files_and_dirs_in(dir_name, extra_files) else raise "Do not know how to handle extra_files: #{extra_files.inspect}" end - Dir.glob(*patterns) + Dir.glob(patterns) end + module_function :all_files_and_dirs_in # Resolves the given symlink into an absolute path. # diff --git a/test/data_sources/test_static.rb b/test/data_sources/test_static.rb index 6d5048cf00..d0bcabccba 100644 --- a/test/data_sources/test_static.rb +++ b/test/data_sources/test_static.rb @@ -32,7 +32,7 @@ def test_items_with_symlinks # Check all files expected_filenames = ['foo/a.png', 'foo/1/b.png', 'foo/c.png'].sort - actual_filenames = Nanoc::Extra::FilesystemTools.all_files_in('foo').sort + actual_filenames = Nanoc::Extra::FilesystemTools.all_files_in('foo', nil).sort assert_equal expected_filenames, actual_filenames # Check items diff --git a/test/extra/test_filesystem_tools.rb b/test/extra/test_filesystem_tools.rb index d6187aec39..cd777c61fe 100644 --- a/test/extra/test_filesystem_tools.rb +++ b/test/extra/test_filesystem_tools.rb @@ -32,7 +32,7 @@ def test_all_files_in_follows_symlinks_to_dirs 'dir0/sub/sub/sub/sub/sub/sub/sub/sub/sub/foo.md', 'dir0/sub/sub/sub/sub/sub/sub/sub/sub/sub/sub/foo.md' ] - actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir0').sort + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir0', nil).sort assert_equal expected_files, actual_files end @@ -47,7 +47,7 @@ def test_all_files_in_follows_symlinks_to_dirs_too_many end assert_raises Nanoc::Extra::FilesystemTools::MaxSymlinkDepthExceededError do - Nanoc::Extra::FilesystemTools.all_files_in('dir0') + Nanoc::Extra::FilesystemTools.all_files_in('dir0', nil) end end @@ -61,7 +61,7 @@ def test_all_files_in_relativizes_directory_names File.symlink('../bar', 'foo/barlink') expected_files = ['foo/barlink/y.md', 'foo/x.md'] - actual_files = Nanoc::Extra::FilesystemTools.all_files_in('foo').sort + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('foo', nil).sort assert_equal expected_files, actual_files end @@ -74,7 +74,7 @@ def test_all_files_in_follows_symlinks_to_files # Check expected_files = ['dir/bar-link', 'dir/foo'] - actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir').sort + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir', nil).sort assert_equal expected_files, actual_files end @@ -106,7 +106,7 @@ def test_dotfiles_are_valid_items FileUtils.mkdir_p('dir') File.open('dir/.htaccess', 'w') { |io| io.write('o hai') } - actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir').sort + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir', nil).sort assert_equal ['dir/.htaccess'], actual_files end From 4a6da47851d74f4f807a4116ec7aa59470d20c0d Mon Sep 17 00:00:00 2001 From: Michal Papis Date: Sun, 23 Nov 2014 22:55:26 +0100 Subject: [PATCH 4/8] improved documentation and errors for all_files_and_dirs_in --- lib/nanoc/extra/filesystem_tools.rb | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/nanoc/extra/filesystem_tools.rb b/lib/nanoc/extra/filesystem_tools.rb index 1da117a197..08b4b545f6 100644 --- a/lib/nanoc/extra/filesystem_tools.rb +++ b/lib/nanoc/extra/filesystem_tools.rb @@ -43,10 +43,14 @@ def initialize(filename) # @param [String] dir_name The name of the directory whose contents to # fetch # + # @param [String, Array, nil] extra_files The list of extra patterns + # to extend the file search for files not found by default, example + # "**/.{htaccess,htpasswd}" which is also the default for `nil` + # # @param [Integer] recursion_limit The maximum number of times to # recurse into a symlink to a directory # - # @return [Array] A list of filenames + # @return [Array] A list of file names # # @raise [MaxSymlinkDepthExceededError] if too many indirections are # encountered while resolving symlinks @@ -80,6 +84,19 @@ def all_files_in(dir_name, extra_files, recursion_limit = 10) end module_function :all_files_in + # Returns all files and directories in the given directory and + # directories below it. + # + # @param [String] dir_name The name of the directory whose contents to + # fetch + # + # @param [String, Array, nil] extra_files The list of extra patterns + # to extend the file search for files not found by default, example + # "**/.{htaccess,htpasswd}" which is also the default for `nil` + # + # @return [Array] A list of files and directories + # + # @raise [GenericTrivial] when pattern can not be handled def all_files_and_dirs_in(dir_name, extra_files) patterns = [ "#{dir_name}/**/*" ] case extra_files @@ -90,7 +107,8 @@ def all_files_and_dirs_in(dir_name, extra_files) when Array patterns += extra_files.map { |extra_file| "#{dir_name}/#{extra_file}" } else - raise "Do not know how to handle extra_files: #{extra_files.inspect}" + raise Nanoc::Errors::GenericTrivial, + "Do not know how to handle extra_files: #{extra_files.inspect}" end Dir.glob(patterns) end From bc8073984e8c8e26add3fc4520ca9bb18ba0202e Mon Sep 17 00:00:00 2001 From: Michal Papis Date: Sun, 23 Nov 2014 23:04:16 +0100 Subject: [PATCH 5/8] add more tests for all_files_in --- test/extra/test_filesystem_tools.rb | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/extra/test_filesystem_tools.rb b/test/extra/test_filesystem_tools.rb index cd777c61fe..bdbfb58a0a 100644 --- a/test/extra/test_filesystem_tools.rb +++ b/test/extra/test_filesystem_tools.rb @@ -101,6 +101,15 @@ def test_resolve_symlink_too_many end end + def test_unwanted_dotfiles_not_found + # Write sample files + FileUtils.mkdir_p('dir') + File.open('dir/.DS_Store', 'w') { |io| io.write('o hai') } + + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir', nil).sort + assert_equal [], actual_files + end + def test_dotfiles_are_valid_items # Write sample files FileUtils.mkdir_p('dir') @@ -110,4 +119,35 @@ def test_dotfiles_are_valid_items assert_equal ['dir/.htaccess'], actual_files end + def test_user_dotfiles_are_valid_items + # Write sample files + FileUtils.mkdir_p('dir') + File.open('dir/.other', 'w') { |io| io.write('o hai') } + + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir', "**/.other").sort + assert_equal ['dir/.other'], actual_files + end + + def test_multiple_user_dotfiles_are_valid_items + # Write sample files + FileUtils.mkdir_p('dir') + File.open('dir/.other', 'w') { |io| io.write('o hai') } + File.open('dir/.DS_Store', 'w') { |io| io.write('o hai') } + + actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir', ["**/.other", "**/.DS_Store"]).sort + assert_equal ['dir/.other', 'dir/.DS_Store'].sort, actual_files.sort + end + + def test_unknown_pattern + # Write sample files + FileUtils.mkdir_p('dir') + File.open('dir/.other', 'w') { |io| io.write('o hai') } + + pattern = {:dotfiles => "**/.other"} + + assert_raises Nanoc::Errors::GenericTrivial, "Do not know how to handle extra_files: #{pattern.inspect}" do + Nanoc::Extra::FilesystemTools.all_files_in('dir0', pattern) + end + end + end From f097c88b5902917e53ffcdba8bfb9914a7536fad Mon Sep 17 00:00:00 2001 From: Michal Papis Date: Mon, 1 Dec 2014 02:13:44 +0100 Subject: [PATCH 6/8] remove the default ht files list --- lib/nanoc/extra/filesystem_tools.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/nanoc/extra/filesystem_tools.rb b/lib/nanoc/extra/filesystem_tools.rb index 08b4b545f6..fa57be27df 100644 --- a/lib/nanoc/extra/filesystem_tools.rb +++ b/lib/nanoc/extra/filesystem_tools.rb @@ -45,7 +45,7 @@ def initialize(filename) # # @param [String, Array, nil] extra_files The list of extra patterns # to extend the file search for files not found by default, example - # "**/.{htaccess,htpasswd}" which is also the default for `nil` + # "**/.{htaccess,htpasswd}" # # @param [Integer] recursion_limit The maximum number of times to # recurse into a symlink to a directory @@ -92,7 +92,7 @@ def all_files_in(dir_name, extra_files, recursion_limit = 10) # # @param [String, Array, nil] extra_files The list of extra patterns # to extend the file search for files not found by default, example - # "**/.{htaccess,htpasswd}" which is also the default for `nil` + # "**/.{htaccess,htpasswd}" # # @return [Array] A list of files and directories # @@ -101,7 +101,6 @@ def all_files_and_dirs_in(dir_name, extra_files) patterns = [ "#{dir_name}/**/*" ] case extra_files when nil - patterns << "#{dir_name}/**/.{htaccess,htpasswd}" when String patterns << "#{dir_name}/#{extra_files}" when Array From c5b46f921026f8ae349a045b78f90ab4d94bb4ce Mon Sep 17 00:00:00 2001 From: Michal Papis Date: Mon, 1 Dec 2014 03:04:34 +0100 Subject: [PATCH 7/8] fix (local) tests for rubocop and removed defaults --- lib/nanoc/extra/filesystem_tools.rb | 2 +- test/extra/test_filesystem_tools.rb | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/nanoc/extra/filesystem_tools.rb b/lib/nanoc/extra/filesystem_tools.rb index fa57be27df..5a6472a29d 100644 --- a/lib/nanoc/extra/filesystem_tools.rb +++ b/lib/nanoc/extra/filesystem_tools.rb @@ -98,7 +98,7 @@ def all_files_in(dir_name, extra_files, recursion_limit = 10) # # @raise [GenericTrivial] when pattern can not be handled def all_files_and_dirs_in(dir_name, extra_files) - patterns = [ "#{dir_name}/**/*" ] + patterns = ["#{dir_name}/**/*"] case extra_files when nil when String diff --git a/test/extra/test_filesystem_tools.rb b/test/extra/test_filesystem_tools.rb index bdbfb58a0a..4a091bbbb5 100644 --- a/test/extra/test_filesystem_tools.rb +++ b/test/extra/test_filesystem_tools.rb @@ -105,18 +105,10 @@ def test_unwanted_dotfiles_not_found # Write sample files FileUtils.mkdir_p('dir') File.open('dir/.DS_Store', 'w') { |io| io.write('o hai') } - - actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir', nil).sort - assert_equal [], actual_files - end - - def test_dotfiles_are_valid_items - # Write sample files - FileUtils.mkdir_p('dir') File.open('dir/.htaccess', 'w') { |io| io.write('o hai') } actual_files = Nanoc::Extra::FilesystemTools.all_files_in('dir', nil).sort - assert_equal ['dir/.htaccess'], actual_files + assert_equal [], actual_files end def test_user_dotfiles_are_valid_items From 1782d70b0c76adcaf0e354c2d759eb91e3a1bb54 Mon Sep 17 00:00:00 2001 From: Michal Papis Date: Sat, 6 Dec 2014 15:57:43 +0100 Subject: [PATCH 8/8] use concat instead of += --- lib/nanoc/extra/filesystem_tools.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nanoc/extra/filesystem_tools.rb b/lib/nanoc/extra/filesystem_tools.rb index 5a6472a29d..59d4e04767 100644 --- a/lib/nanoc/extra/filesystem_tools.rb +++ b/lib/nanoc/extra/filesystem_tools.rb @@ -104,7 +104,7 @@ def all_files_and_dirs_in(dir_name, extra_files) when String patterns << "#{dir_name}/#{extra_files}" when Array - patterns += extra_files.map { |extra_file| "#{dir_name}/#{extra_file}" } + patterns.concat(extra_files.map { |extra_file| "#{dir_name}/#{extra_file}" }) else raise Nanoc::Errors::GenericTrivial, "Do not know how to handle extra_files: #{extra_files.inspect}"