Add Rackspace Cloud Files input.

lib/logstash/inputs/cloud_files.rb

@@ -0,0 +1,69 @@
+require 'fog'
+require 'stringio'
+require 'zlib'
+require 'logstash/inputs/base'
+require 'logstash/namespace'
+
+class LogStash::Inputs::CloudFiles < LogStash::Inputs::Base
+  milestone 1
+  config_name 'cloud_files'
+
+  default :codec, 'line'
+
+  config :username, :validate => :string, :required => true
+  config :api_key, :validate => :string, :required => true
+  config :region, :validate => :string, :required => true
+  config :container, :validate => :string, :required => true
+  config :interval, :validate => :number, :default => 60
+  config :sincedb_path, :validate => :string, :required => true
+
+  def register
+    @api = Fog::Storage.new(:provider => 'Rackspace', :rackspace_username => @username, :rackspace_api_key => @api_key, :rackspace_region => @region)
+  end
+
+  def run(queue)
+    loop do
+      process_log_files(queue)
+      sleep(@interval)
+    end
+
+    finished
+  end
+
+  private
+
+  def process_log_files(queue)
+    last_read = sincedb_read
+
+    container = @api.directories.get(@container)
+    container.files.each do |file|
+      process_file(queue, file) if file.last_modified > last_read
+    end
+  end
+
+  def process_file(queue, file)
+    log_stream = StringIO.new(file.body)
+    reader = Zlib::GzipReader.new(log_stream)
+
+    @codec.decode(reader.read) do |event|
+      decorate(event)
+      queue << event
+    end
+
+    sincedb_write(file.last_modified)
+  end
+
+  def sincedb_read
+    if File.exists?(@sincedb_path)
+      since = Time.parse(File.read(@sincedb_path))
+    else
+      since = Time.new(0)
+    end
+
+    since
+  end
+
+  def sincedb_write(since)
+    File.open(@sincedb_path, 'w') { |f| f.write(since.to_s) }
+  end
+end

logstash.gemspec

@@ -35,7 +35,7 @@ Gem::Specification.new do |gem|
   # Input/Output/Filter dependencies
   #TODO Can these be optional?
   gem.add_runtime_dependency "awesome_print"                    #(MIT license)
-  gem.add_runtime_dependency "aws-sdk"                          #{Apache 2.0 license}  
+  gem.add_runtime_dependency "aws-sdk"                          #{Apache 2.0 license}
   gem.add_runtime_dependency "google-api-client"                #{Apache 2.0 license}
   gem.add_runtime_dependency "heroku"                           #(MIT license)
   gem.add_runtime_dependency "addressable"                      #(Apache 2.0 license)
@@ -77,6 +77,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "twitter", "5.0.0.rc.1"            #(MIT license)
   gem.add_runtime_dependency "rsolr"                            #(Apache 2.0 license)
   gem.add_runtime_dependency "edn"                              #(MIT license)
+  gem.add_runtime_dependency "fog"                              #(MIT License)
 
   if RUBY_PLATFORM == 'java'
     gem.platform = RUBY_PLATFORM
@@ -119,7 +120,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "rspec"            #(MIT license)
   gem.add_runtime_dependency "insist", "1.0.0"  #(Apache 2.0 license)
   gem.add_runtime_dependency "rumbster"         # For faking smtp in email tests (Apache 2.0 license)
-  
+
   #Development Deps
   gem.add_development_dependency "coveralls"