@@ -24,7 +24,7 @@ class NewsArticleVersion < ActiveRecord::Base validates_presence_of :news_article before_validation :setup_text after_save :update_text - + # populate the object from a NewsPage object def populate_from_page(page) self.text_hash = page.hash @@ -39,32 +39,82 @@ class NewsArticleVersion < ActiveRecord::Base self.id <=> b.id end end - + def text @text ||= news_article_version_text.to_s end - + + def to_xapian_doc + XapianFu::XapianDoc.new(:id => id, :title => title, :text => text, + :created_at => created_at) + end + def text=(new_text) @text_changed = true if @text != new_text @text = new_text end - + + def self.xapian_search(query, options = { }) + xapian_db_ro.ro.reopen + docs = xapian_db_ro.search(query, options) + doc_hash = { } + docs.each { |d| doc_hash[d.id] = d } + versions = find(doc_hash.keys) + versions.sort_by do |v| + doc_hash[v.id].weight + end.reverse + end + + def self.xapian_db_ro + @xapian_db_ro ||= XapianFu::XapianDb.new(:dir => File.join(RAILS_ROOT, 'xapian/news_article_versions'), + :sortable => :created_at) + end + private - + + def self.xapian_db + @xapian_db ||= XapianFu::XapianDb.new(:dir => File.join(RAILS_ROOT, 'xapian/news_article_versions'), + :create => true, :sortable => :created_at, :index_positions => false) + end + def setup_text build_news_article_version_text unless news_article_version_text true end - + + def self.xapian_rebuild(options = { }) + options = { :batch_size => 1000 }.merge(options) + puts logger.info("starting xapian_rebuild for NewsArticleVersion with options #{options.inspect}") + find_in_batches(options) do |batch| + xapian_batch_index(batch) + end + end + def update_text if @text_changed news_article_version_text.update_attributes(:text => @text) end true end - + + def self.xapian_batch_index(records) + bm = Benchmark.measure do + records.each { |nv| xapian_db << nv.to_xapian_doc } + end + puts logger.info("#{records.size} versions (#{records.first.id}..#{records.last.id}) indexed in %.2f seconds (#{(records.size/bm.total).round}/second)" % bm.total) + end + + def self.xapian_update + logger.info("starting xapian_update for NewsArticleVersion") + last = xapian_db.documents.max(:id) + xapian_rebuild(:conditions => ['news_article_versions.id > ?', last.id]) + rescue Exception => e + xapian_db.flush + raise e + end + def set_new_version self.version = news_article.versions_count end - + end app/models/news_article_version.rb @@ -17,16 +17,16 @@ <tr> <% if version.news_article.versions_count > 1 -%> <% if version.version == 0 -%> - <td><%= link_to h(truncate(version.title, 75)), { :action => :diff, :id => version.news_article.id, + <td><%= link_to h(truncate(version.title, :length => 95)), { :action => :diff, :id => version.news_article.id, :version_a => version.version+1, :version_b => version.version }, :title => "#{h(version.title)}, first published #{version.news_article.created_at}" %></td> <% else -%> - <td><%= link_to h(truncate(version.title, 75)), { :action => :diff, :id => version.news_article.id, + <td><%= link_to h(truncate(version.title, :length => 75)), { :action => :diff, :id => version.news_article.id, :version_a => version.version, :version_b => version.version-1 }, :title => "#{h(version.title)}, first published #{version.news_article.created_at}" %></td> <% end -%> <% else -%> -<td><span title="<%= "#{h(version.title)}, first published #{version.news_article.created_at}" %>"><%= h(truncate(version.news_article.title, 75)) %></span></td> +<td><span title="<%= "#{h(version.title)}, first published #{version.news_article.created_at}" %>"><%= h(truncate(version.news_article.title, :length => 75)) %></span></td> <% end -%> <td><%= version.created_at.strftime('%d %b %Y') %></td> <td class="version-and-total"><%= version.version %> of <%= version.news_article.versions_count %></td> @@ -35,5 +35,5 @@ <% end %> </table> <hr/> -<%= will_paginate @versions %> +<%= will_paginate @versions if @versions.respond_to?(:total_pages) %> <%end -%> app/views/news_articles/search.rhtml @@ -7,7 +7,7 @@ <th>Source</th></tr> <% @versions.each do |version| %> <tr> -<% if version.version > 1 %> +<% if version.version > 0 %> <td> <%= link_to h(truncate(version.title, :length => 75)), diff_url(version.news_article.id, version.version - 1, version.version), :title => h(version.title) %> @@ -24,7 +24,7 @@ <%= version.votes %> </td> <td> - <small><%= version.news_article.source %></small> + <small><%#= version.source %></small> </td> </tr> <% end %> app/views/versions/index.html.erb @@ -7,5 +7,10 @@ <%= @version.text %> </td> <td> +<ul> +<% @xap_doc.terms.sort_by(&:wdf)[0..40].each do |term| %> +<li><%= term.term %></li> +<% end %> +</ul> </td> </table> \ No newline at end of file app/views/versions/show.html.erb @@ -34,6 +34,7 @@ Rails::Initializer.run do |config| config.gem 'web-page-parser' config.gem "diff-lcs", :lib => "diff/lcs" + config.gem "xapian-fu", :lib => 'xapian_fu' # Only load the plugins named here, in the order given. By default, all plugins @@ -88,3 +89,4 @@ require 'diff_html' require 'http' require 'zlib' include HTTP +require 'xapian_fu' config/environment.rb cc72ffa2fa60e9af8cd184e30caaee341c3cfd0e 07334c203c350cd294e9ac29249f2ace38bc70e6 John Leach john@johnleach.co.uk http://github.com/johnl/news-sniffer/commit/4490393669013c340bd51efa94d0151ac3c025e9 4490393669013c340bd51efa94d0151ac3c025e9 2009-07-06T13:41:51-07:00 2009-07-06T13:35:49-07:00 Merge branch 'xapian' Conflicts: app/controllers/news_articles_controller.rb app/models/news_article_version.rb app/views/news_articles/search.rhtml app/views/versions/index.html.erb config/routes.rb 30e4fcbc801e4305c17393ce0b78565e1c89e2d6 John Leach john@johnleach.co.uk