fetch and cache text of stories

useful for reading the summary of a page when the site is down, and
allows us to show the content of the story in the rss feed as well
as making its content searchable.

whether this is legal/responsible is up for debate, so this is not
being committed on master yet.

app/assets/stylesheets/application.css

@@ -528,6 +528,13 @@ div.story_text {
div.story_text p {
	margin: 0.75em 0;
}
div.collapsed_story_text {
	display: none;
}
a#story_text_expander {
	display: block;
	font-style: italic;
}

div.comment_text {
	max-width: 700px;

app/models/story.rb

@@ -19,6 +19,7 @@ class Story < ActiveRecord::Base
  attr_accessible :title, :description, :tags_a, :moderation_reason

  before_create :assign_short_id
  before_create :fetch_story_cache
  before_save :log_moderation, :check_tags
  after_create :mark_submitter
  after_save :deal_with_tags
@@ -27,6 +28,7 @@ class Story < ActiveRecord::Base
    indexes url
    indexes title
    indexes description
    indexes story_cache
    indexes user.username, :as => :author
    indexes tags(:tag), :as => :tags

@@ -239,6 +241,12 @@ def fetched_content(for_remote_ip = nil)
    @fetched_content
  end

  def fetch_story_cache
    if self.url.present?
      self.story_cache = Diffbot.get_url_text(self.url)
    end
  end

  def calculated_hotness
    score = upvotes - downvotes
    order = Math.log([ score.abs, 1 ].max, 10)

app/views/home/rss.erb

@@ -17,6 +17,9 @@
        <% if story.markeddown_description.present? %>
          <description><%= raw coder.encode(story.markeddown_description,
            :decimal) %></description>
        <% elsif story.story_cache.present? %>
          <description><%= raw coder.encode(simple_format(story.story_cache),
            :decimal) %></description>
        <% end %>
        <% story.taggings.each do |tagging| %>
          <category><%= tagging.tag.tag %></category>

app/views/stories/show.html.erb

@@ -1,14 +1,30 @@
<ol class="stories">
	<%= render :partial => "stories/listdetail",
  <%= render :partial => "stories/listdetail",
    :locals => { :story => @story } %>
</ol>

<div class="story_content">
	<% if @story.markeddown_description.present? %>
		<div class="story_text">
			<%= raw @story.markeddown_description %>
		</div>
	<% end %>
  <% if @story.markeddown_description.present? %>
    <div class="story_text">
      <%= raw @story.markeddown_description %>
    </div>
  <% elsif @story.story_cache.present? %>
    <a id="story_text_expander">View Cached Story Text</a>
    <div class="story_text collapsed_story_text">
      <p>
      <em>All story content copyright of its respective owner.</em>
      </p>
      <%= simple_format(@story.story_cache) %>
    </div>

    <script>
      $("#story_text_expander").click(function() {
        $(".story_text").css({ "display": "block" });
        $("#story_text_expander").remove();
        return false;
      });
    </script>
  <% end %>

  <p></p>
  <% if @user && !@story.is_gone? && !@story.previewing %>

db/migrate/20121004153529_add_story_text.rb

@@ -0,0 +1,9 @@
class AddStoryText < ActiveRecord::Migration
  def up
    add_column :stories, :story_cache, :text
  end

  def down
    remove_column :stories, :story_cache
  end
end

db/schema.rb

@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.

ActiveRecord::Schema.define(:version => 20120919195401) do
ActiveRecord::Schema.define(:version => 20121004153529) do

  create_table "comments", :force => true do |t|
    t.datetime "created_at",                                                                          :null => false
@@ -89,6 +89,7 @@
    t.integer  "is_moderated",           :limit => 1,                                   :default => 0,   :null => false
    t.decimal  "hotness",                               :precision => 20, :scale => 10, :default => 0.0, :null => false
    t.text     "markeddown_description"
    t.text     "story_cache"
  end

  add_index "stories", ["hotness"], :name => "hotness_idx"

extras/diffbot.rb

@@ -0,0 +1,32 @@
class Diffbot
  cattr_accessor :API_KEY

  # this needs to be overridden in config/initializers/production.rb
  @@API_KEY = nil

  API_URL = "http://www.diffbot.com/api/article"

  def self.get_url_text(url)
    if !@@API_KEY
      return
    end

    db_url = "#{API_URL}?token=#{@@API_KEY}&url=#{CGI.escape(url)}"

    begin
      s = Sponge.new
      s.timeout = 4
      res = s.fetch(db_url)
      if res.present?
        j = JSON.parse(res)

        # turn newlines into double newlines, so they become paragraphs
        return j["text"].gsub("\n", "\n\n")
      end
    rescue => e
      Rails.logger.error "error fetching #{db_url}: #{e.message}"
    end

    nil
  end
end