Merge pull request #3545 from delftswa2014/site_extract_readers

Merge pull request 3545

lib/jekyll.rb

@@ -49,12 +49,19 @@ module Jekyll
   autoload :Filters,             'jekyll/filters'
   autoload :FrontmatterDefaults, 'jekyll/frontmatter_defaults'
   autoload :Layout,              'jekyll/layout'
-  autoload :LayoutReader,        'jekyll/layout_reader'
+  autoload :CollectionReader,    'jekyll/readers/collection_reader'
+  autoload :DataReader,          'jekyll/readers/data_reader'
+  autoload :LayoutReader,        'jekyll/readers/layout_reader'
+  autoload :DraftReader,         'jekyll/readers/draft_reader'
+  autoload :PostReader,          'jekyll/readers/post_reader'
+  autoload :PageReader,          'jekyll/readers/page_reader'
+  autoload :StaticFileReader,    'jekyll/readers/static_file_reader'
   autoload :LogAdapter,          'jekyll/log_adapter'
   autoload :Page,                'jekyll/page'
   autoload :PluginManager,       'jekyll/plugin_manager'
   autoload :Post,                'jekyll/post'
   autoload :Publisher,           'jekyll/publisher'
+  autoload :Reader,              'jekyll/reader'
   autoload :Regenerator,         'jekyll/regenerator'
   autoload :RelatedPosts,        'jekyll/related_posts'
   autoload :Renderer,            'jekyll/renderer'

lib/jekyll/reader.rb

@@ -0,0 +1,121 @@
+# encoding: UTF-8
+require 'csv'
+
+module Jekyll
+  class Reader
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+    end
+
+    # Read Site data from disk and load it into internal data structures.
+    #
+    # Returns nothing.
+    def read
+      @site.layouts = LayoutReader.new(site).read
+      read_directories
+      @site.data = DataReader.new(site).read(site.config['data_source'])
+      CollectionReader.new(site).read
+    end
+
+    # Recursively traverse directories to find posts, pages and static files
+    # that will become part of the site according to the rules in
+    # filter_entries.
+    #
+    # dir - The String relative path of the directory to read. Default: ''.
+    #
+    # Returns nothing.
+    def read_directories(dir = '')
+      base = site.in_source_dir(dir)
+
+      dot = Dir.chdir(base) { filter_entries(Dir.entries('.'), base) }
+      dot_dirs = dot.select{ |file| File.directory?(@site.in_source_dir(base,file)) }
+      dot_files = (dot - dot_dirs)
+      dot_pages = dot_files.select{ |file| Utils.has_yaml_header?(@site.in_source_dir(base,file)) }
+      dot_static_files = dot_files - dot_pages
+
+      retrieve_posts(dir)
+      retrieve_dirs(base, dir, dot_dirs)
+      retrieve_pages(dir, dot_pages)
+      retrieve_static_files(dir, dot_static_files)
+    end
+
+    # Retrieves all the posts(posts/drafts) from the given directory
+    # and add them to the site and sort them.
+    #
+    # dir - The String representing the directory to retrieve the posts from.
+    #
+    # Returns nothing.
+    def retrieve_posts(dir)
+      site.posts.concat(PostReader.new(site).read(dir))
+      site.posts.concat(DraftReader.new(site).read(dir)) if site.show_drafts
+      site.posts.sort!
+    end
+
+    # Recursively traverse directories with the read_directories function.
+    #
+    # base - The String representing the site's base directory.
+    # dir - The String representing the directory to traverse down.
+    # dot_dirs - The Array of subdirectories in the dir.
+    #
+    # Returns nothing.
+    def retrieve_dirs(base, dir, dot_dirs)
+      dot_dirs.map { |file|
+        dir_path = site.in_source_dir(dir,file)
+        rel_path = File.join(dir, file)
+        @site.reader.read_directories(rel_path) unless @site.dest.sub(/\/$/, '') == dir_path
+      }
+    end
+
+    # Retrieve all the pages from the current directory,
+    # add them to the site and sort them.
+    #
+    # dir - The String representing the directory retrieve the pages from.
+    # dot_pages - The Array of pages in the dir.
+    #
+    # Returns nothing.
+    def retrieve_pages(dir, dot_pages)
+      site.pages.concat(PageReader.new(site, dir).read(dot_pages))
+      site.pages.sort_by!(&:name)
+    end
+
+    # Retrieve all the static files from the current directory,
+    # add them to the site and sort them.
+    #
+    # dir - The directory retrieve the static files from.
+    # dot_static_files - The static files in the dir.
+    #
+    # Returns nothing.
+    def retrieve_static_files(dir, dot_static_files)
+      site.static_files.concat(StaticFileReader.new(site, dir).read(dot_static_files))
+      site.static_files.sort_by!(&:relative_path)
+    end
+
+    # Filter out any files/directories that are hidden or backup files (start
+    # with "." or "#" or end with "~"), or contain site content (start with "_"),
+    # or are excluded in the site configuration, unless they are web server
+    # files such as '.htaccess'.
+    #
+    # entries - The Array of String file/directory entries to filter.
+    # base_directory - The string representing the optional base directory.
+    #
+    # Returns the Array of filtered entries.
+    def filter_entries(entries, base_directory = nil)
+      EntryFilter.new(site, base_directory).filter(entries)
+    end
+
+    # Read the entries from a particular directory for processing
+    #
+    # dir - The String representing the relative path of the directory to read.
+    # subfolder - The String representing the directory to read.
+    #
+    # Returns the list of entries to process
+    def get_entries(dir, subfolder)
+      base = site.in_source_dir(dir, subfolder)
+      return [] unless File.exist?(base)
+      entries = Dir.chdir(base) { filter_entries(Dir['**/*'], base) }
+      entries.delete_if { |e| File.directory?(site.in_source_dir(base, e)) }
+    end
+  end
+end

lib/jekyll/readers/collection_reader.rb

@@ -0,0 +1,19 @@
+module Jekyll
+  class CollectionReader
+    attr_reader :site, :content
+    def initialize(site)
+      @site = site
+      @content = {}
+    end
+
+    # Read in all collections specified in the configuration
+    #
+    # Returns nothing.
+    def read
+      site.collections.each do |_, collection|
+        collection.read unless collection.label.eql?('data')
+      end
+    end
+
+  end
+end

lib/jekyll/readers/data_reader.rb

@@ -0,0 +1,69 @@
+module Jekyll
+  class DataReader
+    attr_reader :site, :content
+    def initialize(site)
+      @site = site
+      @content = {}
+    end
+
+    # Read all the files in <source>/<dir>/_drafts and create a new Draft
+    # object with each one.
+    #
+    # dir - The String relative path of the directory to read.
+    #
+    # Returns nothing.
+    def read(dir)
+      base = site.in_source_dir(dir)
+      read_data_to(base, @content)
+      @content
+    end
+
+    # Read and parse all yaml files under <dir> and add them to the
+    # <data> variable.
+    #
+    # dir - The string absolute path of the directory to read.
+    # data - The variable to which data will be added.
+    #
+    # Returns nothing
+    def read_data_to(dir, data)
+      return unless File.directory?(dir) && (!site.safe || !File.symlink?(dir))
+
+      entries = Dir.chdir(dir) do
+        Dir['*.{yaml,yml,json,csv}'] + Dir['*'].select { |fn| File.directory?(fn) }
+      end
+
+      entries.each do |entry|
+        path = @site.in_source_dir(dir, entry)
+        next if File.symlink?(path) && site.safe
+
+        key = sanitize_filename(File.basename(entry, '.*'))
+        if File.directory?(path)
+          read_data_to(path, data[key] = {})
+        else
+          data[key] = read_data_file(path)
+        end
+      end
+    end
+
+    # Determines how to read a data file.
+    #
+    # Returns the contents of the data file.
+    def read_data_file(path)
+      case File.extname(path).downcase
+        when '.csv'
+          CSV.read(path, {
+                           :headers => true,
+                           :encoding => site.config['encoding']
+                       }).map(&:to_hash)
+        else
+          SafeYAML.load_file(path)
+      end
+    end
+
+    def sanitize_filename(name)
+      name.gsub!(/[^\w\s-]+/, '')
+      name.gsub!(/(^|\b\s)\s+($|\s?\b)/, '\\1\\2')
+      name.gsub(/\s+/, '_')
+    end
+  end
+end

lib/jekyll/readers/draft_reader.rb

@@ -0,0 +1,37 @@
+module Jekyll
+  class DraftReader
+    attr_reader :site, :unfiltered_content
+    def initialize(site)
+      @site = site
+      @unfiltered_content = Array.new
+    end
+
+    # Read all the files in <source>/<dir>/_drafts and create a new Draft
+    # object with each one.
+    #
+    # dir - The String relative path of the directory to read.
+    #
+    # Returns nothing.
+    def read(dir)
+      @unfiltered_content = read_content(dir, '_drafts')
+      @unfiltered_content.select{ |draft| site.publisher.publish?(draft) }
+    end
+
+    # Read all the content files from <source>/<dir>/magic_dir
+    #   and return them with the type klass.
+    #
+    # dir - The String relative path of the directory to read.
+    # magic_dir - The String relative directory to <dir>,
+    #   looks for content here.
+    # klass - The return type of the content.
+    #
+    # Returns klass type of content files
+    def read_content(dir, magic_dir)
+      @site.reader.get_entries(dir, magic_dir).map do |entry|
+        Draft.new(site, site.source, dir, entry) if Draft.valid?(entry)
+      end.reject do |entry|
+        entry.nil?
+      end
+    end
+  end
+end

lib/jekyll/readers/page_reader.rb

@@ -0,0 +1,21 @@
+module Jekyll
+  class PageReader
+    attr_reader :site, :dir, :unfiltered_content
+    def initialize(site, dir)
+      @site = site
+      @dir = dir
+      @unfiltered_content = Array.new
+    end
+
+    # Read all the files in <source>/<dir>/ for Yaml header and create a new Page
+    # object for each file.
+    #
+    # dir - The String relative path of the directory to read.
+    #
+    # Returns an array of static pages.
+    def read(files)
+      files.map{ |page| @unfiltered_content << Page.new(@site, @site.source, @dir, page) }
+      @unfiltered_content.select{ |page| site.publisher.publish?(page) }
+    end
+  end
+end

lib/jekyll/readers/post_reader.rb

@@ -0,0 +1,37 @@
+module Jekyll
+  class PostReader
+    attr_reader :site, :unfiltered_content
+    def initialize(site)
+      @site = site
+      @unfiltered_content = Array.new
+    end
+
+    # Read all the files in <source>/<dir>/_posts and create a new Post
+    # object with each one.
+    #
+    # dir - The String relative path of the directory to read.
+    #
+    # Returns nothing.
+    def read(dir)
+      @unfiltered_content = read_content(dir, '_posts')
+      @unfiltered_content.select{ |post| site.publisher.publish?(post) }
+    end
+
+    # Read all the content files from <source>/<dir>/magic_dir
+    #   and return them with the type klass.
+    #
+    # dir - The String relative path of the directory to read.
+    # magic_dir - The String relative directory to <dir>,
+    #   looks for content here.
+    # klass - The return type of the content.
+    #
+    # Returns klass type of content files
+    def read_content(dir, magic_dir)
+      @site.reader.get_entries(dir, magic_dir).map do |entry|
+        Post.new(site, site.source, dir, entry) if Post.valid?(entry)
+      end.reject do |entry|
+        entry.nil?
+      end
+    end
+  end
+end

lib/jekyll/readers/static_file_reader.rb

@@ -0,0 +1,21 @@
+module Jekyll
+  class StaticFileReader
+    attr_reader :site, :dir, :unfiltered_content
+    def initialize(site, dir)
+      @site = site
+      @dir = dir
+      @unfiltered_content = Array.new
+    end
+
+    # Read all the files in <source>/<dir>/ for Yaml header and create a new Page
+    # object for each file.
+    #
+    # dir - The String relative path of the directory to read.
+    #
+    # Returns an array of static files.
+    def read(files)
+      files.map{ |file| @unfiltered_content << StaticFile.new(@site, @site.source, @dir, file)}
+      @unfiltered_content
+    end
+  end
+end