Permalink
Browse files

archive old logs periodically

  • Loading branch information...
1 parent 93406b2 commit e7ea41a26e46370d03ddaf286f092e6ee503ac87 @svenfuchs svenfuchs committed Jan 24, 2013
Showing with 50 additions and 37 deletions.
  1. +1 −0 .gitignore
  2. +1 −1 Gemfile
  3. +18 −22 Gemfile.lock
  4. +11 −1 lib/travis/logs/aggregate.rb
  5. +1 −13 lib/travis/logs/app.rb
  6. +18 −0 lib/travis/logs/archive.rb
View
@@ -1,3 +1,4 @@
config/database.yml
config/travis.yml
config/travis.pro.yml
+play
View
@@ -2,7 +2,7 @@ source :rubygems
ruby '1.9.3', engine: 'jruby', engine_version: '1.7.0'
-gem 'travis-core', github: 'travis-ci/travis-core'
+gem 'travis-core', github: 'travis-ci/travis-core', branch: 'sf-archive-logs'
gem 'travis-support', github: 'travis-ci/travis-support'
gem 'travis-sidekiqs', github: 'travis-ci/travis-sidekiqs', require: nil
View
@@ -15,11 +15,11 @@ GIT
GIT
remote: git://github.com/travis-ci/travis-core.git
- revision: 59d88d712ca2f99133b30580e5e03a1bfce23122
+ revision: ebcae5c92715ee494aa817be41d5fc2f3ea86353
specs:
travis-core (0.0.1)
- actionmailer (~> 3.2.3)
- activerecord (~> 3.2.3)
+ actionmailer (~> 3.2.11)
+ activerecord (~> 3.2.11)
coder (~> 0.3.0)
data_migrations (~> 0.0.1)
gh
@@ -28,8 +28,8 @@ GIT
multi_json
postmark-rails (~> 0.4.1)
pusher (~> 0.11.0)
- railties (~> 3.2.3)
- rake (~> 0.9.2.2)
+ railties (~> 3.2.11)
+ rake
redis (~> 3.0)
rollout (~> 1.1.0)
simple_states (~> 0.1.1)
@@ -44,7 +44,7 @@ GIT
GIT
remote: git://github.com/travis-ci/travis-support.git
- revision: a214c215e1b52bdf4d3ab5d31e7e2df52fad5a0c
+ revision: ff712aca1083a588974f835a84c574e6976aeb29
specs:
travis-support (0.0.1)
@@ -89,7 +89,7 @@ GEM
arel (3.0.2)
atomic (1.0.1-java)
avl_tree (1.1.3)
- backports (2.6.5)
+ backports (2.7.1)
bouncy-castle-java (1.5.0146.1)
builder (3.0.4)
celluloid (0.12.4)
@@ -98,7 +98,7 @@ GEM
coder (0.3.0)
coderay (1.0.8)
connection_pool (1.0.0)
- crack (0.3.1)
+ crack (0.3.2)
data_migrations (0.0.1)
activerecord
rake
@@ -108,7 +108,7 @@ GEM
facter (1.6.17)
faraday (0.8.4)
multipart-post (~> 1.1)
- gh (0.9.0)
+ gh (0.9.1)
addressable
backports (~> 2.3)
faraday (~> 0.8)
@@ -127,12 +127,12 @@ GEM
hitimes (1.1.1-java)
hot_bunnies (1.3.8-java)
i18n (0.6.1)
- jdbc-postgres (9.2.1002)
+ jdbc-postgres (9.2.1002.1)
journey (1.0.4)
jruby-openssl (0.7.7)
bouncy-castle-java (>= 1.5.0146.1)
json (1.7.6-java)
- listen (0.7.1)
+ listen (0.7.2)
lumberjack (1.0.2)
mail (2.4.4)
i18n (>= 0.4.0)
@@ -153,26 +153,22 @@ GEM
net-http-pipeline (1.0.1)
newrelic_rpm (3.3.5)
polyglot (0.3.3)
- postmark (0.9.15)
+ postmark (0.9.18)
json
rake
postmark-rails (0.4.1)
actionmailer
postmark (>= 0.9.0)
rake
- pry (0.9.10)
- coderay (~> 1.0.5)
- method_source (~> 0.8)
- slop (~> 3.3.1)
pry (0.9.10-java)
coderay (~> 1.0.5)
method_source (~> 0.8)
- slop (~> 3.3.1)
+ slop (~> 3.4)
spoon (~> 0.0)
pusher (0.11.0)
multi_json (~> 1.0)
signature (~> 0.1.4)
- rack (1.4.3)
+ rack (1.4.4)
rack-cache (1.2)
rack (>= 0.4)
rack-ssl (1.3.2)
@@ -186,7 +182,7 @@ GEM
rake (>= 0.8.7)
rdoc (~> 3.4)
thor (>= 0.14.6, < 2.0)
- rake (0.9.2.2)
+ rake (10.0.3)
rdoc (3.12)
json (~> 1.4)
redis (3.0.2)
@@ -200,7 +196,7 @@ GEM
rspec-expectations (2.7.0)
diff-lcs (~> 1.1.2)
rspec-mocks (2.7.0)
- sidekiq (2.6.4)
+ sidekiq (2.6.5)
celluloid (~> 0.12.0)
connection_pool (~> 1.0)
multi_json (~> 1)
@@ -210,7 +206,7 @@ GEM
simple_states (0.1.1)
activesupport
hashr (~> 0.0.10)
- slop (3.3.3)
+ slop (3.4.3)
spoon (0.0.1)
sprockets (2.2.2)
hike (~> 1.2)
@@ -219,7 +215,7 @@ GEM
tilt (~> 1.1, != 1.3.0)
thor (0.14.6)
tilt (1.3.3)
- timers (1.0.2)
+ timers (1.1.0)
treetop (1.4.12)
polyglot
polyglot (>= 0.3.1)
@@ -1,18 +1,28 @@
require 'travis'
require 'travis/support'
+require 'travis/logs/archive'
require 'core_ext/kernel/run_periodically'
Travis::Database.connect
Travis::Features.start
Travis::Exceptions::Reporter.start
-Travis::Instrumentation.setup
+
+Travis::Async.enabled = true
+Travis::Async::Sidekiq.setup(Travis.config.redis.url, Travis.config.sidekiq)
+
+instrumenter = Travis.env == 'production' ? Travis::Instrumentation : Travis::Notification
+instrumenter.setup
def aggregate_logs
Travis.run_service(:logs_aggregate)
rescue Exception => e
Travis::Exceptions.handle(e)
end
+run_periodically(3) do
+ archive_logs if Travis::Features.feature_active?(:log_archiving)
+end
+
run_periodically(Travis.config.logs.intervals.vacuum || 10) do
aggregate_logs if Travis::Features.feature_active?(:log_aggregation)
end.join
View
@@ -10,17 +10,6 @@
require 'travis/task'
-# TODO why the hell does the setter below not work
-module Travis
- class Task
- class << self
- def run_local?
- true
- end
- end
- end
-end
-
module Travis
module Logs
autoload :Handler, 'travis/logs/handler'
@@ -59,8 +48,7 @@ def preload_constants
def setup
Travis::Async.enabled = true
Travis::Amqp.config = Travis.config.amqp
- Travis::Task.run_local = true # don't pipe log updates through travis_tasks
- # Travis::Async::Sidekiq.setup(Travis.config.redis.url, Travis.config.sidekiq)
+ Travis::Addons::Pusher::Task.run_local = true # don't pipe log updates through travis_tasks
Travis::Features.start
Travis::Database.connect
View
@@ -0,0 +1,18 @@
+LOG_ARCHIVE_INTERVAL = 60 * 60 * 24 * 365
+
+def unarchived_log
+ sql = "archived_at IS NULL AND created_at <= NOW() - interval '? seconds'"
+ Artifact::Log.where(sql, LOG_ARCHIVE_INTERVAL).order(:id).select(:id).first
+end
+
+def archive_logs
+ if log = unarchived_log
+ puts "queueing archive task for log: #{log.id}"
+ Travis::Addons::Archive::Task.run(:archive, type: 'log', id: log.id)
+ else
+ puts 'DONE ARCHIVING JOBS'
+ Thread.current.exit
+ end
+rescue Exception => e
+ Travis::Exceptions.handle(e)
+end

0 comments on commit e7ea41a

Please sign in to comment.