Skip to content
This repository
Browse code

archive old logs periodically

  • Loading branch information...
commit e7ea41a26e46370d03ddaf286f092e6ee503ac87 1 parent 93406b2
Sven Fuchs authored January 24, 2013
1  .gitignore
... ...
@@ -1,3 +1,4 @@
1 1
 config/database.yml
2 2
 config/travis.yml
3 3
 config/travis.pro.yml
  4
+play
2  Gemfile
@@ -2,7 +2,7 @@ source :rubygems
2 2
 
3 3
 ruby '1.9.3', engine: 'jruby', engine_version: '1.7.0'
4 4
 
5  
-gem 'travis-core',        github: 'travis-ci/travis-core'
  5
+gem 'travis-core',        github: 'travis-ci/travis-core', branch: 'sf-archive-logs'
6 6
 gem 'travis-support',     github: 'travis-ci/travis-support'
7 7
 gem 'travis-sidekiqs',    github: 'travis-ci/travis-sidekiqs', require: nil
8 8
 
40  Gemfile.lock
@@ -15,11 +15,11 @@ GIT
15 15
 
16 16
 GIT
17 17
   remote: git://github.com/travis-ci/travis-core.git
18  
-  revision: 59d88d712ca2f99133b30580e5e03a1bfce23122
  18
+  revision: ebcae5c92715ee494aa817be41d5fc2f3ea86353
19 19
   specs:
20 20
     travis-core (0.0.1)
21  
-      actionmailer (~> 3.2.3)
22  
-      activerecord (~> 3.2.3)
  21
+      actionmailer (~> 3.2.11)
  22
+      activerecord (~> 3.2.11)
23 23
       coder (~> 0.3.0)
24 24
       data_migrations (~> 0.0.1)
25 25
       gh
@@ -28,8 +28,8 @@ GIT
28 28
       multi_json
29 29
       postmark-rails (~> 0.4.1)
30 30
       pusher (~> 0.11.0)
31  
-      railties (~> 3.2.3)
32  
-      rake (~> 0.9.2.2)
  31
+      railties (~> 3.2.11)
  32
+      rake
33 33
       redis (~> 3.0)
34 34
       rollout (~> 1.1.0)
35 35
       simple_states (~> 0.1.1)
@@ -44,7 +44,7 @@ GIT
44 44
 
45 45
 GIT
46 46
   remote: git://github.com/travis-ci/travis-support.git
47  
-  revision: a214c215e1b52bdf4d3ab5d31e7e2df52fad5a0c
  47
+  revision: ff712aca1083a588974f835a84c574e6976aeb29
48 48
   specs:
49 49
     travis-support (0.0.1)
50 50
 
@@ -89,7 +89,7 @@ GEM
89 89
     arel (3.0.2)
90 90
     atomic (1.0.1-java)
91 91
     avl_tree (1.1.3)
92  
-    backports (2.6.5)
  92
+    backports (2.7.1)
93 93
     bouncy-castle-java (1.5.0146.1)
94 94
     builder (3.0.4)
95 95
     celluloid (0.12.4)
@@ -98,7 +98,7 @@ GEM
98 98
     coder (0.3.0)
99 99
     coderay (1.0.8)
100 100
     connection_pool (1.0.0)
101  
-    crack (0.3.1)
  101
+    crack (0.3.2)
102 102
     data_migrations (0.0.1)
103 103
       activerecord
104 104
       rake
@@ -108,7 +108,7 @@ GEM
108 108
     facter (1.6.17)
109 109
     faraday (0.8.4)
110 110
       multipart-post (~> 1.1)
111  
-    gh (0.9.0)
  111
+    gh (0.9.1)
112 112
       addressable
113 113
       backports (~> 2.3)
114 114
       faraday (~> 0.8)
@@ -127,12 +127,12 @@ GEM
127 127
     hitimes (1.1.1-java)
128 128
     hot_bunnies (1.3.8-java)
129 129
     i18n (0.6.1)
130  
-    jdbc-postgres (9.2.1002)
  130
+    jdbc-postgres (9.2.1002.1)
131 131
     journey (1.0.4)
132 132
     jruby-openssl (0.7.7)
133 133
       bouncy-castle-java (>= 1.5.0146.1)
134 134
     json (1.7.6-java)
135  
-    listen (0.7.1)
  135
+    listen (0.7.2)
136 136
     lumberjack (1.0.2)
137 137
     mail (2.4.4)
138 138
       i18n (>= 0.4.0)
@@ -153,26 +153,22 @@ GEM
153 153
     net-http-pipeline (1.0.1)
154 154
     newrelic_rpm (3.3.5)
155 155
     polyglot (0.3.3)
156  
-    postmark (0.9.15)
  156
+    postmark (0.9.18)
157 157
       json
158 158
       rake
159 159
     postmark-rails (0.4.1)
160 160
       actionmailer
161 161
       postmark (>= 0.9.0)
162 162
       rake
163  
-    pry (0.9.10)
164  
-      coderay (~> 1.0.5)
165  
-      method_source (~> 0.8)
166  
-      slop (~> 3.3.1)
167 163
     pry (0.9.10-java)
168 164
       coderay (~> 1.0.5)
169 165
       method_source (~> 0.8)
170  
-      slop (~> 3.3.1)
  166
+      slop (~> 3.4)
171 167
       spoon (~> 0.0)
172 168
     pusher (0.11.0)
173 169
       multi_json (~> 1.0)
174 170
       signature (~> 0.1.4)
175  
-    rack (1.4.3)
  171
+    rack (1.4.4)
176 172
     rack-cache (1.2)
177 173
       rack (>= 0.4)
178 174
     rack-ssl (1.3.2)
@@ -186,7 +182,7 @@ GEM
186 182
       rake (>= 0.8.7)
187 183
       rdoc (~> 3.4)
188 184
       thor (>= 0.14.6, < 2.0)
189  
-    rake (0.9.2.2)
  185
+    rake (10.0.3)
190 186
     rdoc (3.12)
191 187
       json (~> 1.4)
192 188
     redis (3.0.2)
@@ -200,7 +196,7 @@ GEM
200 196
     rspec-expectations (2.7.0)
201 197
       diff-lcs (~> 1.1.2)
202 198
     rspec-mocks (2.7.0)
203  
-    sidekiq (2.6.4)
  199
+    sidekiq (2.6.5)
204 200
       celluloid (~> 0.12.0)
205 201
       connection_pool (~> 1.0)
206 202
       multi_json (~> 1)
@@ -210,7 +206,7 @@ GEM
210 206
     simple_states (0.1.1)
211 207
       activesupport
212 208
       hashr (~> 0.0.10)
213  
-    slop (3.3.3)
  209
+    slop (3.4.3)
214 210
     spoon (0.0.1)
215 211
     sprockets (2.2.2)
216 212
       hike (~> 1.2)
@@ -219,7 +215,7 @@ GEM
219 215
       tilt (~> 1.1, != 1.3.0)
220 216
     thor (0.14.6)
221 217
     tilt (1.3.3)
222  
-    timers (1.0.2)
  218
+    timers (1.1.0)
223 219
     treetop (1.4.12)
224 220
       polyglot
225 221
       polyglot (>= 0.3.1)
12  lib/travis/logs/aggregate.rb
... ...
@@ -1,11 +1,17 @@
1 1
 require 'travis'
2 2
 require 'travis/support'
  3
+require 'travis/logs/archive'
3 4
 require 'core_ext/kernel/run_periodically'
4 5
 
5 6
 Travis::Database.connect
6 7
 Travis::Features.start
7 8
 Travis::Exceptions::Reporter.start
8  
-Travis::Instrumentation.setup
  9
+
  10
+Travis::Async.enabled = true
  11
+Travis::Async::Sidekiq.setup(Travis.config.redis.url, Travis.config.sidekiq)
  12
+
  13
+instrumenter = Travis.env == 'production' ? Travis::Instrumentation : Travis::Notification
  14
+instrumenter.setup
9 15
 
10 16
 def aggregate_logs
11 17
   Travis.run_service(:logs_aggregate)
@@ -13,6 +19,10 @@ def aggregate_logs
13 19
   Travis::Exceptions.handle(e)
14 20
 end
15 21
 
  22
+run_periodically(3) do
  23
+  archive_logs if Travis::Features.feature_active?(:log_archiving)
  24
+end
  25
+
16 26
 run_periodically(Travis.config.logs.intervals.vacuum || 10) do
17 27
   aggregate_logs if Travis::Features.feature_active?(:log_aggregation)
18 28
 end.join
14  lib/travis/logs/app.rb
@@ -10,17 +10,6 @@
10 10
 
11 11
 require 'travis/task'
12 12
 
13  
-# TODO why the hell does the setter below not work
14  
-module Travis
15  
-  class Task
16  
-    class << self
17  
-      def run_local?
18  
-        true
19  
-      end
20  
-    end
21  
-  end
22  
-end
23  
-
24 13
 module Travis
25 14
   module Logs
26 15
     autoload :Handler, 'travis/logs/handler'
@@ -59,8 +48,7 @@ def preload_constants
59 48
         def setup
60 49
           Travis::Async.enabled = true
61 50
           Travis::Amqp.config = Travis.config.amqp
62  
-          Travis::Task.run_local = true # don't pipe log updates through travis_tasks
63  
-          # Travis::Async::Sidekiq.setup(Travis.config.redis.url, Travis.config.sidekiq)
  51
+          Travis::Addons::Pusher::Task.run_local = true # don't pipe log updates through travis_tasks
64 52
 
65 53
           Travis::Features.start
66 54
           Travis::Database.connect
18  lib/travis/logs/archive.rb
... ...
@@ -0,0 +1,18 @@
  1
+LOG_ARCHIVE_INTERVAL = 60 * 60 * 24 * 365
  2
+
  3
+def unarchived_log
  4
+  sql = "archived_at IS NULL AND created_at <= NOW() - interval '? seconds'"
  5
+  Artifact::Log.where(sql, LOG_ARCHIVE_INTERVAL).order(:id).select(:id).first
  6
+end
  7
+
  8
+def archive_logs
  9
+  if log = unarchived_log
  10
+    puts "queueing archive task for log: #{log.id}"
  11
+    Travis::Addons::Archive::Task.run(:archive, type: 'log', id: log.id)
  12
+  else
  13
+    puts 'DONE ARCHIVING JOBS'
  14
+    Thread.current.exit
  15
+  end
  16
+rescue Exception => e
  17
+  Travis::Exceptions.handle(e)
  18
+end

0 notes on commit e7ea41a

Please sign in to comment.
Something went wrong with that request. Please try again.