simplify the log receiving app

Gemfile.lock

@@ -15,7 +15,7 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-core.git
-  revision: 987188a5f96f5d7e4f6a6e314ab402f3fbad0963
+  revision: b8d0d63f433c8e63472b97b3c287b99d31027ecb
   specs:
     travis-core (0.0.1)
       actionmailer (~> 3.2.11)

bin/aggregate_logs

@@ -1,6 +1,6 @@
 #! /usr/bin/env ruby
 
-$: << File.expand_path('../../lib', __FILE__)
+$: << 'lib'
 
 require 'bundler/setup'
 require 'travis/logs/aggregate'

bin/receive_logs

@@ -1,10 +1,15 @@
 #! /usr/bin/env ruby
 
-$: << File.expand_path('../../lib', __FILE__)
+$: << 'lib'
 
 require 'bundler/setup'
-require 'travis/logs/app'
+require 'travis/logs/receive'
+require 'core_ext/module/load_constants'
 
-app = Travis::Logs::App.new
-app.start
+[Travis::Logs::Receive, Travis].each do |target|
+  target.load_constants!(skip: [/::AssociationCollection$/])
+end
 
+app = Travis::Logs::Receive.new
+app.setup
+app.run

lib/travis/logs/receive.rb

@@ -0,0 +1,65 @@
+require 'multi_json'
+
+require 'travis'
+require 'core_ext/module/load_constants'
+require 'timeout'
+require 'sidekiq'
+
+$stdout.sync = true
+
+module Travis
+  module Logs
+    class Receive
+      autoload :Queue, 'travis/logs/receive/queue'
+
+      def setup
+        Travis::Async.enabled = true
+        Travis::Amqp.config = Travis.config.amqp
+        Travis::Addons::Pusher::Task.run_local = true # don't pipe log updates through travis_tasks
+
+        Travis::Database.connect
+        Travis::Exceptions::Reporter.start
+        Travis::Notification.setup
+        Travis::Addons.register
+
+        Travis::LogSubscriber::ActiveRecordMetrics.attach
+        Travis::Memory.new(:logs).report_periodically if Travis.env == 'production'
+      end
+
+      def run
+        Queue.subscribe(queue_name, &method(:route))
+        0.upto(shards).each do |shard|
+          Queue.subscribe(queue_name(shard), &method(:receive))
+        end
+      end
+
+      private
+
+        def route(payload)
+          shard = payload['id'].to_i % shards
+          queue = queue_name(shard)
+          payload.update(uuid: Travis.uuid)
+          Travis::Amqp::Publisher.jobs(queue).publish(payload)
+        end
+
+        def receive(payload)
+          Travis.run_service(:logs_append, data: payload)
+        end
+
+        def queue_name(shard = nil)
+          name = ['logs']
+          name << number      if number
+          name << ".#{shard}" if shard
+          name.join
+        end
+
+        def shards
+          Travis.config.logs.shards - 1
+        end
+
+        def number
+           ENV['LOGS_QUEUE']
+        end
+    end
+  end
+end

lib/travis/logs/receive/queue.rb

@@ -0,0 +1,57 @@
+require 'coder'
+
+module Travis
+  module Logs
+    class Receive
+      class Queue
+        include Logging
+
+        def self.subscribe(name, &handler)
+          new(name, &handler).subscribe
+        end
+
+        attr_reader :name, :handler
+
+        def initialize(name, &handler)
+          @name = name
+          @handler = handler
+        end
+
+        def subscribe
+          Travis::Amqp::Consumer.jobs(name).subscribe(:ack => true, declare: true, &method(:receive))
+        end
+
+        private
+
+          def receive(message, payload)
+            failsafe(message, payload) do
+              payload = decode(payload) || raise("no payload #{message.inspect}")
+              Travis.uuid = payload.delete('uuid')
+              handler.call(payload)
+            end
+          end
+
+          def failsafe(message, payload, options = {}, &block)
+            Timeout::timeout(options[:timeout] || 60, &block)
+          rescue Exception => e
+            begin
+              puts e.message, e.backtrace
+              Travis::Exceptions.handle(Hub::Error.new(message.properties.type, payload, e))
+            rescue Exception => e
+              puts "!!!FAILSAFE!!! #{e.message}", e.backtrace
+            end
+          ensure
+            message.ack
+          end
+
+          def decode(payload)
+            cleaned = Coder.clean(payload)
+            MultiJson.decode(cleaned)
+          rescue StandardError => e
+            error "[decode error] payload could not be decoded with engine #{MultiJson.engine.to_s}: #{e.inspect} #{payload.inspect}"
+            nil
+          end
+      end
+    end
+  end
+end

lib/travis/logs/sidekiq/archive.rb

@@ -1,16 +1,13 @@
+# Do not use. This was used to archive the backlog of logs.
+# For day to day archiving travis-tasks should be used.
+
 require 'travis'
 require 'sidekiq'
 require 'core_ext/hash/deep_symbolize_keys'
-# require 'core_ext/module/load_constants'
 
 Travis::Database.connect
 ActiveRecord::Base.logger.level = Logger::ERROR
 Travis::Notification.setup
-# Travis::Exceptions::Reporter.start
-
-# Sidekiq::Logging.logger.formatter = ->(level, _, _, msg) do
-#   "TID-#{Thread.current.object_id.to_s(36)} #{level}: #{msg}\n"
-# end
 
 Sidekiq.configure_server do |c|
   c.redis = { url: Travis.config.redis.url }