lib/caboose.rb lib/caboose/spider_integrator.rb lib/caboose/spider_integrator/spiderable_form.rb @@ -2,10 +2,13 @@ if defined?(Merb::Plugins) $LOAD_PATH << File.dirname(__FILE__) + require 'hpricot' + require 'caboose' + require 'caboose/spider_integrator' %w(cookie cookie_jar simple_rsync archiver).each do |filename| require "merb_static/#{filename}" end - + # Merb gives you a Merb::Plugins.config hash...feel free to put your stuff in your piece of it Merb::Plugins.config[:merb_static] = { :urls => ["/"], lib/merb_static.rb @@ -1,7 +1,9 @@ module Merb module Static class Archiver - + + include ::Caboose::SpiderIntegrator + attr_accessor :config def initialize @@ -11,7 +13,7 @@ module Merb def self.build new do |a| a.read_configuration - a.copy_assets +# a.copy_assets a.unleash_the_spiders end end @@ -40,11 +42,12 @@ module Merb @config[:urls].each do |relative_url| Merb.logger.info "Fetching #{relative_url}" + # TODO Do this in the request method absolute_url = [domain, relative_url].join Merb.logger.info "Absolute url: #{absolute_url}" response = request(absolute_url) - + if response.status == 200 Merb.logger.info "Response is #{response.status}" # TODO Get file type and extension from headers or request. @@ -56,6 +59,8 @@ module Merb File.open(filename_on_disk, 'w') do |f| f.write(response.body.to_s) end + + spider(response.body, "/", :verbose => true) else raise "Error while fetching page: #{response.inspect}" end @@ -64,6 +69,7 @@ module Merb end def request(uri, env = {}) + # TODO Combine with domain here to get absolute URL uri = url(uri) if uri.is_a?(Symbol) uri = URI(uri) uri.scheme ||= "http" lib/merb_static/archiver.rb 7fd71c4ec2b93a1bdf19b9287807979537c8db61 Geoffrey Grosenbach boss@topfunky.com http://github.com/topfunky/merb_static/commit/3e5b560da6fb590b21a7fc81fca66b5dcc4c5ea8 3e5b560da6fb590b21a7fc81fca66b5dcc4c5ea8 2008-12-15T08:35:15-08:00 2008-12-15T08:35:15-08:00 Added caboose spider_integrator. Modified it for Merb and Hpricot. Implemented initial static asset caching and dynamic HTML caching. e5add8ba536b9fd59d0110e6d0cdeddb7c841d15 Geoffrey Grosenbach boss@topfunky.com