Browse files

Proof of concept

  • Loading branch information...
1 parent aa091a2 commit 727cf0e17b5db6238ba92225b4f6e03cfdba661e @chewbranca chewbranca committed Jun 3, 2011
View
22 app/controllers/jobs_controller.rb
@@ -0,0 +1,22 @@
+class JobsController < ApplicationController
+ def run
+ Rails.logger.debug "Initializing job request"
+ job = Newscloud::Redcloud.q_pop 'asdf'
+ #worker = Resque::Worker.new '*'
+ #worker.work 0
+ #render :text => "Finished up" and return
+ timeout = 10
+ begin
+ if FeedsWorker.respond_to? :set_timeout
+ FeedsWorker.set_timeout timeout
+ end
+ FeedsWorker.perform
+ rescue Newscloud::Redcloud::JobTimeoutError => error
+ #Newscloud::Redcloud.q_push 'asdf', job
+ Rails.logger.debug "***Job not completed in time.. requeuing"
+ render :text => "Job paused for later processing.." and return
+ end
+ Rails.logger.debug "Finished processing job"
+ render :text => "Processed: #{job.inspect}" and return
+ end
+end
View
2 app/helpers/jobs_helper.rb
@@ -0,0 +1,2 @@
+module JobsHelper
+end
View
8 app/workers/feeds_worker.rb
@@ -4,10 +4,14 @@ class FeedsWorker
def self.perform(feed_id = nil)
if feed_id
feed = Feed.active.find_by_id(feed_id)
- N2::FeedParser.update_feed Feed.active.find(feed_id) if feed
+ N2::FeedParser.update_feed(Feed.active.find(feed_id), :timeout => @timeout) if feed
else
- N2::FeedParser.update_feeds
+ N2::FeedParser.update_feeds @timeout
end
end
+
+ def self.set_timeout timeout
+ @timeout = timeout
+ end
end
View
9 app/workers/jobs_worker.rb
@@ -0,0 +1,9 @@
+class JobsWorker
+ @queue = :jobs
+
+ # needs to run nightly
+ def self.perform()
+ {:success => "Processed job"}.to_json
+ end
+
+end
View
1 config/routes.rb
@@ -1,5 +1,6 @@
ActionController::Routing::Routes.draw do |map|
map.resources :oauth_consumers,:member=>{:callback=>:get}
+ map.resources :jobs, :collection => { :run => [:get, :post] }
# Set locale and make pretty urls
map.filter 'locale'
View
13 lib/feed_parser.rb
@@ -1,14 +1,16 @@
module N2
class FeedParser
- def self.update_feeds
+ def self.update_feeds timeout = nil
feeds = Feed.find(:all, :conditions => ["deleted_at is null and specialType = ?", "default"])
- feeds.each { |feed| update_feed feed, false }
+ feeds.each { |feed| update_feed feed, :trigger_expire_cache => false, :timeout => timeout }
expire_newswire_cache
end
- def self.update_feed(feed, trigger_expire_cache = true)
+ def self.update_feed(feed, opts = {})
return false unless feed
+ timeout = opts[:timeout]
+ trigger_expire_cache = opts[:trigger_expire_cache]
Rails.logger.info "Running feedzirra on #{feed.item_title}"
begin
rss = Feedzirra::Feed.fetch_and_parse(feed.rss)
@@ -23,9 +25,14 @@ def self.update_feed(feed, trigger_expire_cache = true)
feed_date = feed.last_fetched_at
pub_date = rss.last_modified
+ start_time = Time.now
if !feed_date or (pub_date and feed_date < pub_date)
items.each do |item|
Rails.logger.info "\tChecking feed items"
+
+ # Pause processing if we have a timeout and it has expired
+ raise Newscloud::Redcloud::JobTimeoutError if timeout and Time.now - start_time > timeout
+
break if feed_date and item.published and (item.published <= feed_date)
next if Newswire.find_by_title item.title
item_summary = item.summary || item.content
View
5 lib/newscloud/redcloud.rb
@@ -1,6 +1,8 @@
module Newscloud
module Redcloud
+ class JobTimeoutError < StandardError; end
+
def self.create(server = nil, namespace = nil)
server ||= APP_CONFIG['redis']
namespace ||= APP_CONFIG['namespace'] || 'newscloud'
@@ -9,5 +11,8 @@ def self.create(server = nil, namespace = nil)
@redis = Redis::Namespace.new(namespace.to_sym, :redis => redis)
end
+ def self.q_push(queue, data) Resque.push(queue, data) end
+ def self.q_pop(queue) Resque.pop(queue) end
+
end
end
View
10 spec/controllers/jobs_controller_spec.rb
@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe JobsController do
+
+ #Delete this example and add some real ones
+ it "should use JobsController" do
+ controller.should be_an_instance_of(JobsController)
+ end
+
+end
View
11 spec/helpers/jobs_helper_spec.rb
@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe JobsHelper do
+
+ #Delete this example and add some real ones or delete this file
+ it "should be included in the object returned by #helper" do
+ included_modules = (class << helper; self; end).send :included_modules
+ included_modules.should include(JobsHelper)
+ end
+
+end

0 comments on commit 727cf0e

Please sign in to comment.