Permalink
Browse files

Miscellaneous cleanup for the rake task

  • Loading branch information...
1 parent af73fc5 commit c2a6035832055dbfcd93c587b2e8ee0f949a74f2 @jeremyruppel jeremyruppel committed Apr 12, 2012
Showing with 48 additions and 14 deletions.
  1. +10 −11 lib/beso/connection.rb
  2. +9 −1 lib/beso/job.rb
  3. +16 −1 lib/tasks/beso.rake
  4. +13 −1 spec/beso/job_spec.rb
@@ -11,12 +11,16 @@ def initialize( options )
@aws_region = options.delete( :aws_region )
end
+ def get( filename )
+ bucket.files.get filename
+ end
+
def read( filename )
- bucket.files.get( filename ).try( :body ) || ''
+ get( filename ).try( :read ) || ''
end
def write( filename, body )
- if file = read( filename )
+ if file = get( filename )
file.body = body
file.save
else
@@ -51,15 +55,10 @@ def storage
module ClassMethods
def connect( &block )
- aws = AWS.new :access_key => Beso.access_key,
- :secret_key => Beso.secret_key,
- :aws_region => Beso.aws_region
-
- aws.connect!
-
- puts '==> connect'
- puts con.inspect
- puts con.methods.sort - Object.methods
+ yield AWS.new :access_key => Beso.access_key,
+ :secret_key => Beso.secret_key,
+ :bucket_name => Beso.bucket_name,
+ :aws_region => Beso.aws_region
end
end
end
View
@@ -29,15 +29,23 @@ def to_csv( options={} )
@since ||= options.delete :since
+ relation = model_class.where( "#{@timestamp} >= ?", @since || first_timestamp )
+
+ return nil if relation.empty?
+
Beso::CSV.generate( @extra.merge( options ) ) do |csv|
csv << ( required_headers + custom_headers )
- model_class.where( "#{@timestamp} > ?", @since || 0 ).each do |model|
+ relation.each do |model|
csv << ( required_columns( model ) + custom_columns( model ) )
end
end
end
+ def first_timestamp
+ model_class.minimum @timestamp
+ end
+
def last_timestamp
model_class.maximum @timestamp
end
View
@@ -5,20 +5,35 @@ namespace :beso do
raise 'Beso has no jobs to run. Please define some in the beso initializer.' if Beso.jobs.empty?
+ puts '==> Connecting...'
+
Beso.connect do |bucket|
+ puts '==> Connected!'
+
config = YAML.load( bucket.read 'beso.yml' ) || { }
+ puts '==> Config:'
+ puts config.inspect
+
Beso.jobs.each do |job|
+ puts "==> Processing job: #{job.event.inspect} since #{config[ job.event ]}"
+
csv = job.to_csv :since => config[ job.event ]
+ bucket.write "#{job.event}-#{config[ job.event ].to_i}.csv", csv
+
+ puts " ==> #{job.event}-#{config[ job.event ].to_i}.csv saved to S3"
+
config[ job.event ] = job.last_timestamp
- bucket.write "#{job.event}-#{config[ job.event ]}.csv", csv
+ puts " ==> New timestamp is #{config[ job.event ]}"
end
bucket.write 'beso.yml', config.to_yaml
+
+ puts '==> Config saved. Donezo.'
end
end
end
View
@@ -2,7 +2,7 @@
describe Beso::Job do
- after do
+ before do
User.destroy_all
end
@@ -157,6 +157,18 @@
EOS
) }
end
+
+ context 'when no records match the query' do
+ subject { Beso::Job.new :message_sent, :table => :users }
+
+ before do
+ User.destroy_all
+ subject.identity { |user| user.id }
+ subject.timestamp :created_at
+ end
+
+ its( :to_csv ){ should be_nil }
+ end
end
describe 'with since specified' do

0 comments on commit c2a6035

Please sign in to comment.