Merge pull request middleman#650 from middleman/sitemap_query

Make Sitemap metadata queryable with arel-style API

middleman-core/features/queryable.feature

@@ -0,0 +1,31 @@
+Feature: Queryable Selector
+  Scenario: Basic Selector Tests
+    Then should initialize with an attribute and an operator
+    Then should raise an exception if the operator is not supported
+  Scenario: Using offset and limit
+    Given the Server is running at "queryable-app"
+    Then should limit the documents to the number specified
+    Then should offset the documents by the number specified
+    Then should support offset and limit at the same time
+    Then should not freak out about an offset higher than the document count
+  Scenario: Using where queries with an equal operator
+    Given the Server is running at "queryable-app"
+    Then should return the right documents
+    Then should be chainable
+    Then should not be confused by attributes not present in all documents
+  Scenario: Using where queries with a complex operator
+    Given the Server is running at "queryable-app"
+    Then with a gt operator should return the right documents
+    Then with a gte operator should return the right documents
+    Then with an in operator should return the right documents
+    Then with an lt operator should return the right documents
+    Then with an lte operator should return the right documents
+    Then with an include operator include should return the right documents
+    Then with mixed operators should return the right documents
+    Then using multiple constrains in one where should return the right documents
+  Scenario: Sorting documents
+    Given the Server is running at "queryable-app"
+    Then should support ordering by attribute ascending
+    Then should support ordering by attribute descending
+    Then should order by attribute ascending by default
+    Then should exclude documents that do not own the attribute

middleman-core/features/step_definitions/queryable_steps.rb

@@ -0,0 +1,123 @@
+Then /^should initialize with an attribute and an operator$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :author, :operator => 'equal'
+  :author.should == selector.attribute
+  'equal'.should == selector.operator
+end
+
+Then /^should raise an exception if the operator is not supported$/ do
+  expect {
+    selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :author, :operator => 'zomg'
+  }.to raise_error(::Middleman::Sitemap::Queryable::OperatorNotSupportedError)
+end
+
+Then /^should limit the documents to the number specified$/ do
+  @server_inst.sitemap.order_by(:id).limit(2).all.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^should offset the documents by the number specified$/ do
+  @server_inst.sitemap.order_by(:id).offset(2).all.map { |r| r.raw_data[:id] }.sort.should == [3,4,5].sort
+end
+
+Then /^should support offset and limit at the same time$/ do
+  @server_inst.sitemap.order_by(:id).offset(1).limit(2).all.map { |r| r.raw_data[:id] }.sort.should == [2,3].sort
+end
+
+Then /^should not freak out about an offset higher than the document count$/ do
+  @server_inst.sitemap.order_by(:id).offset(5).all.should == []
+end
+
+Then /^should return the right documents$/ do
+  documents = @server_inst.sitemap.resources.select { |r| !r.raw_data.empty? }
+  document_1 = documents[0]
+  document_2 = documents[1]
+
+  found_document = @server_inst.sitemap.where(:title => document_1.raw_data[:title]).first
+  document_1.should == found_document
+
+  found_document = @server_inst.sitemap.where(:title => document_2.raw_data[:title]).first
+  document_2.should == found_document
+end
+
+Then /^should be chainable$/ do
+  documents = @server_inst.sitemap.resources.select { |r| !r.raw_data.empty? }
+  document_1 = documents[0]
+
+  document_proxy = @server_inst.sitemap.where(:title => document_1.raw_data[:title])
+  document_proxy.where(:id => document_1.raw_data[:id])
+  document_1.should == document_proxy.first
+end
+
+Then /^should not be confused by attributes not present in all documents$/ do
+  result = @server_inst.sitemap.where(:seldom_attribute => 'is seldom').all
+  result.map { |r| r.raw_data[:id] }.should == [4]
+end
+
+Then /^with a gt operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gt'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [5,3,4].sort
+end
+
+Then /^with a gte operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gte'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [2,5,3,4].sort
+end
+
+Then /^with an in operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'in'
+  found_documents = @server_inst.sitemap.where(selector => [2,3]).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [2,3].sort
+end
+
+Then /^with an lt operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lt'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [1]
+end
+
+Then /^with an lte operator should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lte'
+  found_documents = @server_inst.sitemap.where(selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^with an include operator include should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :tags, :operator => 'include'
+  found_documents = @server_inst.sitemap.where(selector => 'ruby').all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^with mixed operators should return the right documents$/ do
+  in_selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'in'
+  gt_selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'gt'
+  documents_proxy = @server_inst.sitemap.where(in_selector => [2,3])
+  found_documents = documents_proxy.where(gt_selector => 2).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [3]
+end
+
+Then /^using multiple constrains in one where should return the right documents$/ do
+  selector = ::Middleman::Sitemap::Queryable::Selector.new :attribute => :id, :operator => 'lte'
+  found_documents = @server_inst.sitemap.where(selector => 2, :status => :published).all
+  found_documents.map { |r| r.raw_data[:id] }.sort.should == [1,2].sort
+end
+
+Then /^should support ordering by attribute ascending$/ do
+  found_documents = @server_inst.sitemap.order_by(:title => :asc).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [2,3,1,5,4]
+end
+
+Then /^should support ordering by attribute descending$/ do
+  found_documents = @server_inst.sitemap.order_by(:title => :desc).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [4,5,1,3,2]
+end
+
+Then /^should order by attribute ascending by default$/ do
+  found_documents = @server_inst.sitemap.order_by(:title).all
+  found_documents.map { |r| r.raw_data[:id] }.should == [2,3,1,5,4]
+end
+
+Then /^should exclude documents that do not own the attribute$/ do
+  found_documents = @server_inst.sitemap.order_by(:status).all
+  found_documents.map { |r| r.raw_data[:id] }.to_set.should == [1,2].to_set
+end

middleman-core/fixtures/queryable-app/source/2010-08-08-test-document-file.html.markdown

@@ -0,0 +1,8 @@
+---
+id: 1
+title: Some fancy title
+tags: [ruby]
+status: :published
+---
+
+I like being the demo text.

middleman-core/fixtures/queryable-app/source/2010-08-09-another-test-document.html.markdown

@@ -0,0 +1,10 @@
+---
+id: 2
+title: Another title, that's for sure
+tags: [ruby, rails]
+special_attribute: Yes!
+friends: [Anton, Paul]
+status: :published
+---
+
+The body copy.

middleman-core/fixtures/queryable-app/source/2011-12-26-some-test-document.html.markdown

@@ -0,0 +1,6 @@
+---
+id: 5
+title: Some test document
+---
+
+This is just some test document.

middleman-core/fixtures/queryable-app/source/document_with_date_in_yaml.html.markdown

@@ -0,0 +1,7 @@
+---
+id: 3
+title: Document with date in YAML
+date: 2011-04-05
+---
+
+This document has no date in the filename, but in the YAML front matter.

middleman-core/fixtures/queryable-app/source/document_without_date.html.markdown

@@ -0,0 +1,7 @@
+---
+id: 4
+title: This document has no date
+seldom_attribute: is seldom
+---
+
+This document has no date at all.

middleman-core/lib/middleman-core/core_extensions/front_matter.rb

@@ -1,3 +1,5 @@
+require "active_support/core_ext/hash/keys"
+
 # Extensions namespace
 module Middleman::CoreExtensions
 
@@ -34,8 +36,8 @@ def registered(app)
             fmdata = frontmatter_manager.data(path).first || {}
 
             data = {}
-            %w(layout layout_engine).each do |opt|
-              data[opt.to_sym] = fmdata[opt] unless fmdata[opt].nil?
+            [:layout, :layout_engine].each do |opt|
+              data[opt] = fmdata[opt] unless fmdata[opt].nil?
             end
 
             { :options => data, :page => fmdata }
@@ -85,7 +87,7 @@ def parse_yaml_front_matter(content)
           content = content.sub(yaml_regex, "")
 
           begin
-            data = YAML.load($1)
+            data = YAML.load($1).symbolize_keys
           rescue *YAML_ERRORS => e
             logger.error "YAML Exception: #{e.message}"
             return false
@@ -108,7 +110,7 @@ def parse_json_front_matter(content)
 
           begin
             json = ($1+$2).sub(";;;", "{").sub(";;;", "}")
-            data = ActiveSupport::JSON.decode(json)
+            data = ActiveSupport::JSON.decode(json).symbolize_keys
           rescue => e
             logger.error "JSON Exception: #{e.message}"
             return false
@@ -147,7 +149,7 @@ def frontmatter_and_content(path)
           # Probably a binary file, move on
         end
 
-        [::Middleman::Util.recursively_enhance(data).freeze, content]
+        [data, content]
       end
 
       def normalize_path(path)
@@ -187,10 +189,20 @@ def ignored?
 
       # This page's frontmatter
       # @return [Hash]
-      def data
+      def raw_data
         app.frontmatter_manager.data(source_file).first
       end
 
+      def data
+        @_last_raw ||= nil
+        @_last_enhanced ||= nil
+
+        if @_last_raw != raw_data
+          @_last_raw == raw_data
+          @_last_enhanced = ::Middleman::Util.recursively_enhance(raw_data).freeze
+        end
+      end
+
     end
 
     module InstanceMethods