planet/formatter.rb planet/hamlformatter.rb planet/publisher.rb planet/xsltformatter.rb splice.rb test/data/filter/haml/author_email.xml test/data/filter/haml/author_name.xml test/data/filter/haml/author_uri.xml test/data/filter/haml/content_html.xml test/data/filter/haml/content_lang.xml test/data/filter/haml/content_text.xml test/data/filter/haml/content_xhtml.xml test/data/filter/haml/content_xhtml2.xml test/data/filter/haml/enclosure_href.xml test/data/filter/haml/enclosure_length.xml test/data/filter/haml/enclosure_type.xml test/data/filter/haml/feed_feed.ini test/data/filter/haml/feed_generator.ini test/data/filter/haml/feed_link.ini test/data/filter/haml/feed_name.ini test/data/filter/haml/feed_owner_email.ini test/data/filter/haml/feed_owner_name.ini test/data/filter/haml/id.xml test/data/filter/haml/id_only_content.xml test/data/filter/haml/id_only_description.xml test/data/filter/haml/id_only_link.xml test/data/filter/haml/id_only_title.xml test/data/filter/haml/link_href.xml test/data/filter/haml/link_rel.xml test/data/filter/haml/link_type.xml test/data/filter/haml/new_channel.xml test/data/filter/haml/new_channel_date.xml test/data/filter/haml/new_date.xml test/data/filter/haml/planet_name.xml test/data/filter/haml/published.xml test/data/filter/haml/rights.xml test/data/filter/haml/source_author.xml test/data/filter/haml/source_icon.xml test/data/filter/haml/source_id.xml test/data/filter/haml/source_link.xml test/data/filter/haml/source_logo.xml test/data/filter/haml/source_planet_message.xml test/data/filter/haml/source_planet_name.xml test/data/filter/haml/source_rights.xml test/data/filter/haml/source_subtitle.xml test/data/filter/haml/source_title.xml test/data/filter/haml/source_updated.xml test/data/filter/haml/summary_html.xml test/data/filter/haml/summary_lang.xml test/data/filter/haml/summary_text.xml test/data/filter/haml/summary_xhtml.xml test/data/filter/haml/title_html.xml test/data/filter/haml/title_lang.xml test/data/filter/haml/title_text.xml test/data/filter/haml/title_xhtml.xml test/data/filter/haml/updated.xml test/haml.rb themes/common/atom.xml.xslt themes/common/foafroll.xml.xslt themes/common/images/feed-icon-10x10.png themes/common/images/foaf.png themes/common/images/logo.png themes/common/images/opml.png themes/common/images/planet.png themes/common/opml.xml.xslt themes/intertwingly/atom.xml.xslt themes/intertwingly/basic.ini themes/intertwingly/default.css themes/intertwingly/favicon.ico themes/intertwingly/feed-icon-10x10.png themes/intertwingly/index.html.haml themes/intertwingly/personalize.js themes/intertwingly/rss10.xml.haml themes/intertwingly/validate.html.xslt themes/musings/config.ini themes/musings/default.css themes/musings/images/mars.ico themes/musings/images/mars.png themes/musings/index.html.xslt themes/musings/personalize.js @@ -1,9 +1,60 @@ task :default => [:test] +desc "Run the unit tests" task :test do ruby "-rubygems", "test.rb" end +desc "Test for Mars prerequisites" task :prereqs do ruby "-rubygems", "prereqs.rb" end + +desc "Clean up published files" +task :clean do + CLEAN_FILES = FileList['./*.xml', './*.html'] + CLEAN_FILES.each do |fn| + rm fn + end + rmdir "images" +end + +desc "Clean up all files" +task :clean_all do + CLEAN_ALL_FILES = FileList['./*.xml', './*.html', './*.haml', './*.xslt', './*.ini', './*.css', './*.ico', './*.js', './images/*', "source/*", "http/*", "entry/*"] + CLEAN_ALL_FILES.each do |fn| + rm fn + end + rmdir "images" + rmdir "source" + rmdir "http" + rmdir "entry" +end + +desc "Clean up cache files" +task :clean_cache do + CLEAN_CACHE_FILES = FileList[ "source/*", "http/*", "entry/*"] + CLEAN_CACHE_FILES.each do |fn| + rm fn + end + rmdir "source" + rmdir "http" + rmdir "entry" +end + +desc "Install config files for haml/intertwingly example" +task :setup do + FileList['./themes/intertwingly/*'].each do |f| cp "#{f}", "." end + mkdir "images" + mv "feed-icon-10x10.png", "images" +end + +desc "Run planet for haml example" +task :planet do + ruby "planet.rb", "basic.ini" +end + +desc "Splice planet for haml example" +task :splice do + ruby "splice.rb", "basic.ini" +end Rakefile.rb @@ -7,6 +7,13 @@ module Planet @@config end + def Planet.predefine_options(planet_hash) + planet_hash['name'] = "Unconfigured Planet" unless planet_hash['name'] + planet_hash['link'] = '' unless planet_hash['link'] + planet_hash['date_format'] = "%B %d, %Y %I:%M %p" unless planet_hash['date_format'] + planet_hash['new_date_format'] = "%B %d, %Y" unless planet_hash['new_date_format'] + end + # Configuration parser compatible with the data format supported by Python: # http://docs.python.org/lib/module-ConfigParser.html class PythonConfigParser < DelegateClass(Hash) @@ -85,6 +92,8 @@ module Planet planet = self['Planet'] + Planet.predefine_options(planet) + Planet.log_format planet['log_format'] if planet['log_format'] Planet.log_level planet['log_level'] if planet['log_level'] end planet/config.rb @@ -6,7 +6,12 @@ module Planet doc = Planet::Transmogrify.parse(open(source)) doc.attributes['xml:base'] = source - # augment the document with feed parser attributes + Planet.add_attrs(doc) + end + + # Augment a document with feed parser attributes + def Planet.add_attrs doc + class << doc attr_accessor :feed, :entries end @@ -75,7 +80,7 @@ module Planet end class CommonElements < UserDict - text_element :id + text_element :id, :updated, :published alias :guid :id text_construct :rights @@ -111,14 +116,6 @@ module Planet @node.elements.to_a('contributor').map {|node| Author.new(node)} end - def categories - tags.map {|tag| [tag.scheme, tag.term]} - end - - def category - tags.first.term rescue nil - end - def author author_detail.to_s end @@ -145,6 +142,24 @@ module Planet def generator_detail Generator.new(@node.elements['generator']) end + + def message + element = @node.elements['planet:message'] + element ? element.texts.map {|t| t.value}.join : nil + end + + def name + element = @node.elements['planet:name'] + element ? element.texts.map {|t| t.value}.join : nil + end + + def sources + @node.elements.to_a('planet:source').map {|node| Feed.new(node)} + end + + def url + links.select {|link| link.rel=='self'}.first.href rescue nil + end end class Entry < CommonElements @@ -156,6 +171,22 @@ module Planet @node.elements.to_a('content').map {|node| TextConstruct.new(node)} end + def enclosure_href + enclosures.first.href rescue nil + end + + def enclosure_length + enclosures.first.length rescue nil + end + + def enclosure_type + if enclosures.first.is_a?(Planet::Link) + return enclosures.first.type + else + return nil + end + end + def enclosures links.select {|link| link.rel == 'enclosure'} end @@ -214,6 +245,10 @@ module Planet url_norm(@node.xmlbase) end + def language + @node.attributes['xml:lang'] + end + private # DOM to string @@ -255,7 +290,11 @@ module Planet end def to_s - email ? "#{name} (#{email})" : "#{name}" + if name + email ? "#{name} (#{email})" : "#{name}" + else + "#{email}" + end end alias :url :uri planet/harvest.rb @@ -2,6 +2,7 @@ require 'planet/fido' require 'planet/transmogrify' require 'planet/sift' require 'fileutils' +require 'rexml/formatters/default' module Planet @@ -46,7 +47,6 @@ module Planet Planet.log.error "Not a feed - #{uri}" next end - feed.add_text("\n ") end # second set of filters: cardinality, sanitization, dates, and uris @@ -102,6 +102,11 @@ module Planet id ||= entry.elements['link[@rel="alternate"]/@href'] rescue nil next unless id + unless /^\w+\:/ =~ id + id = 'urn:feed-entry-id:' + id + entry.elements['id'].text = id + end + # determine output file name for this entry entry_file = File.join(entry_cache, Planet.filename(id)) @@ -123,7 +128,7 @@ module Planet entry.add(source) if not entry.elements['source'] # output the entry, with a timestamp reflecting the update time - File.open(entry_file, 'w') { |file| file.write(entry.to_s) } + File.open(entry_file, 'w') { |file| REXML::Formatters::Default.new.write(entry, file) } updated = Time.parse(updated.text) File.utime updated, updated, entry_file end @@ -133,7 +138,7 @@ module Planet source.name = 'planet:source' root_attrs.each_pair {|name,value| source.attributes[name]=value} source_file = File.join(source_cache, Planet.filename(sub)) - File.open(source_file, 'w') { |file| file.write(source.to_s) } + File.open(source_file, 'w') { |file| REXML::Formatters::Default.new.write(source, file) } end end end planet/spider.rb @@ -1,8 +1,9 @@ require 'planet/config' -require 'planet/style' require 'planet/xmlparser' require 'fileutils' require 'time' +require 'rexml/formatters/default' +require 'planet/publisher' module Planet @@ -16,18 +17,28 @@ module Planet # produce a minimal feed header (TODO: complete) feed = REXML::Document.new('<feed xmlns="http://www.w3.org/2005/Atom"/>') feed.root.add_namespace('planet', 'http://planet.intertwingly.net/') - id = feed.root.add_element('id') - id.text = URI.join(config['link'],'atom.xml').to_s - link = feed.root.add_element('link', {'rel'=>'self', 'href'=>id.text}) - link.attributes['type'] = 'application/atom+xml' + feed.root.add_namespace('indexing', 'urn:atom-extension:indexing') + feed.root.attributes['indexing:index'] = 'no' title = feed.root.add_element('title') title.text = config['name'].to_s updated = feed.root.add_element('updated') updated.text = Time.now.iso8601 + generator = feed.root.add_element('generator', {'uri' => 'http://github.com/rubys/mars/tree/master'}) + generator.text = 'Mars' + author = feed.root.add_element('author') + author_name = author.add_element('name') + author_name.text = config['owner_name'] + author_email = author.add_element('email') + author_email.text = config['owner_email'] + id = feed.root.add_element('id') + id.text = URI.join(config['link'],'atom.xml').to_s + link_self = feed.root.add_element('link', {'rel'=>'self', 'href'=>id.text}) + link_self.attributes['type'] = 'application/atom+xml' + link_alt = feed.root.add_element('link', {'rel' => 'alternate', 'href' => config['link'] }) + link_alt.attributes['type'] = 'application/xhtml+xml' # add the latest 'items_per_page' entries to the feed entry_cache = File.join(config['cache_directory'],'entry') - feed.root.add_text "\n\n" Dir.chdir(entry_cache) do files = Dir['*'].map {|name| [File.stat(name).mtime.to_i, name]} @@ -54,7 +65,6 @@ module Planet updated.attributes['planet:format'] = formatted_time rescue end - feed.root.add_text "\n\n" end # add source information @@ -80,27 +90,17 @@ module Planet Planet.source(sub, source) unless source.has_elements? feed.root.add source.root - feed.root.add_text "\n" end - # output the Atom feed - File.open(File.join(output_dir,'atom.xml'),'w') do |file| - Planet.log.info 'Producing atom.xml' - feed.write(file) - end +# Let's use an Atom xslt template, instead +# # output the Atom feed +# File.open(File.join(output_dir,'atom.xml'),'w') do |file| +# Planet.log.info 'Producing atom.xml' +# formatter = REXML::Formatters::Default.new +# formatter.write(feed, file) +# end # apply templates - config['template_files'].split.each do |template| - next unless template =~ /^ .* \/ (.*) \. (\w+)/x - - if $2 != 'xslt' - Planet.log.warn "#{$2}: not yet supported" - else - File.open(File.join(output_dir,$1),'w') do |file| - Planet.log.info "Processing template #{template}" - file.write Planet::Xslt.process(template, feed) - end - end - end + TemplatePublisher.new.publish_feed(config['template_files'], feed) end end planet/splice.rb planet/style.rb 99bd28895bd4274bbdb8a6a9c713616504abee12 6c3227414c1c3157eaf93ecf6edd7aff67b36fd5 Sam Ruby rubys@intertwingly.net http://github.com/rubys/mars/commit/ac24693266d447e52f3a8cb02ebbd2d2ed5a9e0f ac24693266d447e52f3a8cb02ebbd2d2ed5a9e0f 2008-09-28T17:47:20-07:00 2008-09-28T17:47:20-07:00 Merge branch 'master' of git@github.com:rubys/mars e0163ea00fa534a1e28242beb1612f262763e2e4 Sam Ruby rubys@intertwingly.net