Permalink
Browse files

Add process counts to units and don't process when more than 3 failures

  • Loading branch information...
1 parent 8e205a0 commit 9104ce55a7207cb8bb3b5a23aabe2b79a9f7a441 @rwdaigle committed Jun 13, 2012
View
@@ -34,7 +34,7 @@ def process!(subject_id, overwrite=false)
end
def process!(overwrite = false)
- (overwrite ? units : units.empty).each do |unit|
+ (overwrite ? units.processable : units.processable.empty).each do |unit|
unit.process!(overwrite)
end
end
View
@@ -1,4 +1,6 @@
class Unit < ActiveRecord::Base
+
+ MAX_PROCESS_COUNT = 2
include Comparable
@@ -33,6 +35,7 @@ class Unit < ActiveRecord::Base
# Get units in random order in a repeateable way (must do SETSEED(seed) first)
scope :random, { :order => "RANDOM()" }
scope :ordered, { :order => 'units.position' }
+ scope :processable, { :conditions => ["process_count <= ?", MAX_PROCESS_COUNT] }
class << self
@@ -57,13 +60,17 @@ def empty?
end
def process!(overwrite = false)
- if overwrite || empty?
+ if (overwrite || empty?) && !max_process_count?
with_processor do |processor_klass|
processor_klass.process!(id)
end
end
end
+ def max_process_count?
+ process_count > MAX_PROCESS_COUNT
+ end
+
def to_log
{ 'unit_id' => self.id, 'unit' => question }
end
@@ -11,9 +11,12 @@ def process!(unit_id)
# Go get it
result = Wordnik.word.get_definitions(unit.question, :limit => 1)
answer = result && !result.empty? ? result[0]['text'] : nil
- answer ? unit.update_attributes(answer: answer) : log({'action' => Event::PROCESS_WORDNIK_NOTFOUND}, unit)
+ log({'action' => Event::PROCESS_WORDNIK_NOTFOUND}, unit) if !answer
- # Log errors
+ # Record
+ unit.update_attributes(answer: answer, :last_processed_at => Time.now.utc, :process_count => (unit.process_count + 1))
+
+ # Log insertion errors
unless unit.valid?
log({'action' => Event::PROCESS_WORDNIK_UPDATE_FAILED}, unit, {'status' => 'error', 'message' => unit.errors.full_messages.join(', ')})
end
@@ -3,9 +3,11 @@ class LogReceiver
class << self
def receive(log_data)
- if(log_data)
- receivers.each do |receiver|
- receiver.call(log_data['action'], log_data)
+ if Rails.env.production?
+ if(log_data)
+ receivers.each do |receiver|
+ receiver.call(log_data['action'], log_data)
+ end
end
end
end
@@ -0,0 +1,13 @@
+class AddUnitProcessingColumns < ActiveRecord::Migration
+
+ def up
+ change_table :units do |t|
+ t.datetime :last_processed_at
+ t.integer :process_count, :default => 0, :null => false
+ end
+ end
+
+ def down
+ remove_column :units, :last_processed_at, :process_count
+ end
+end
View
@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20120602003006) do
+ActiveRecord::Schema.define(:version => 20120613012416) do
create_table "associations", :force => true do |t|
t.integer "user_id", :null => false
@@ -74,13 +74,15 @@
add_index "subjects", ["permalink"], :name => "index_subjects_on_permalink"
create_table "units", :force => true do |t|
- t.string "question", :null => false
+ t.string "question", :null => false
t.text "answer"
- t.integer "subject_id", :null => false
+ t.integer "subject_id", :null => false
t.integer "position"
t.datetime "created_at"
t.datetime "updated_at"
t.string "processor_type"
+ t.datetime "last_processed_at"
+ t.integer "process_count", :default => 0, :null => false
end
add_index "units", ["position"], :name => "index_units_on_position"

0 comments on commit 9104ce5

Please sign in to comment.