Add a sitemap (Closes #44)

.gitignore

@@ -80,3 +80,5 @@ pickle-email-*.html
 
 # Keep newrelic license key secret =)
 config/newrelic.yml
+
+public/sitemaps/

Gemfile

@@ -29,6 +29,9 @@ gem 'redcarpet'
 # Pagination
 gem 'kaminari'
 
+# SEO
+gem 'dynamic_sitemaps'
+
 group :assets do
   gem 'coffee-rails'
 end

Gemfile.lock

@@ -131,6 +131,7 @@ GEM
     docile (1.1.5)
     domain_name (0.5.25)
       unf (>= 0.0.5, < 1.0.0)
+    dynamic_sitemaps (2.0.0)
     email_spec (2.0.0)
       htmlentities (~> 4.3.3)
       launchy (~> 2.1)
@@ -364,6 +365,7 @@ DEPENDENCIES
   cucumber-rails
   cucumber-timecop (= 0.0.3)
   database_cleaner
+  dynamic_sitemaps
   email_spec
   git
   hub

app/controllers/home_controller.rb

@@ -0,0 +1,12 @@
+class HomeController < ApplicationController
+  def sitemap
+    path = Rails.root.join('public', 'sitemaps', 'sitemap.xml')
+    if File.exists?(path)
+      render xml: open(path).read
+    else
+      render text: 'Sitemap not found.', status: :not_found
+    end
+  end
+
+  def robots; end
+end

app/models/homebrew/tools.rb

@@ -16,7 +16,7 @@ def self.extract_class_name(formula)
         GithubGistFormula
       )
       class_name = formula.scan(
-        /^class\s(\w+)\s<\s(?:#{inheriting_classes.join("|")})$/
+        /^(\s+)?class\s(\w+)\s<\s(?:#{inheriting_classes.join("|")})$/
       )
       class_name.flatten.last
     end
@@ -92,6 +92,7 @@ def self.find_or_create_formula!(filename, name)
       formula = Homebrew::Formula.find_by(filename: filename)
       return formula if formula
 
+      Rails.logger.info "Creating formula '#{name}' (#{filename})"
       Homebrew::Formula.create!(filename: filename, name: name, external: true)
     end
   end

app/views/home/robots.text.erb

@@ -0,0 +1 @@
+Sitemap: <%= sitemap_url %>

app/workers/homebrew_formula_import_worker.rb

@@ -6,6 +6,7 @@
 class HomebrewFormulaImportWorker
   include Sidekiq::Worker
   include Sidetiq::Schedulable
+  sidekiq_options backtrace: true
 
   attr_accessor :formula
 
@@ -65,7 +66,9 @@ def perform
                       "#{error.message}"
   ensure
     @import.ended_at = Time.now
-    unless @import.save
+    if @import.save
+      DynamicSitemaps.generate_sitemap
+    else
       Rails.logger.warn 'Unable to update the import with ID ' \
                         "#{@import.id} : #{@import.errors.full_message}"
     end

config/routes.rb

@@ -8,6 +8,9 @@
 
   resources :documentation, only: :index
 
+  get 'sitemap.xml' => 'home#sitemap', format: :xml, as: :sitemap
+  get 'robots.:format' => 'home#robots', format: :text, as: :robots
+
   resources :formulas, only: [:index, :show], path: '' do
     member do
       get 'refresh_description'

config/sitemap.rb

@@ -0,0 +1,46 @@
+# Change this to your host.
+# See the readme at https://github.com/lassebunk/dynamic_sitemaps
+# for examples of multiple hosts and folders.
+host 'brewformulas.org'
+
+sitemap :site do
+  url root_url, last_mod: Time.now, change_freq: 'daily', priority: 1.0
+end
+
+# You can have multiple sitemaps like the above – just make sure their names
+# are different.
+
+# Automatically link to all pages using the routes specified
+# using "resources :pages" in config/routes.rb. This will also
+# automatically set <lastmod> to the date and time in page.updated_at:
+#
+sitemap_for Homebrew::Formula.internals.active, name: :formula do |formula|
+  url formula_url(formula.name)
+end
+
+# For products with special sitemap name and priority, and link to comments:
+#
+# sitemap_for Product.published, name: :published_products do |product|
+#   url product, last_mod: product.updated_at,
+#                priority: (product.featured? ? 1.0 : 0.7)
+#   url product_comments_url(product)
+# end
+
+# If you want to generate multiple sitemaps in different folders
+# (for example if you have more than one domain), you can specify a folder
+# before the sitemap definitions:
+#
+#   Site.all.each do |site|
+#     folder "sitemaps/#{site.domain}"
+#     host site.domain
+#
+#     sitemap :site do
+#       url root_url
+#     end
+#
+#     sitemap_for site.products.scoped
+#   end
+
+# Ping search engines after sitemap generation:
+#
+#   ping_with "http://#{host}/sitemap.xml"