store mails in json format for push to server

Rakefile

@@ -4,7 +4,7 @@ require 'rake'
 begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
-    gem.name = "sonar-imap-pull-connector"
+    gem.name = "sonar_imap_pull_connector"
     gem.summary = %Q{imap connector for sonar}
     gem.description = %Q{an imap pull connector for sonar, retrieving and deleting mails from an imap mailbox}
     gem.email = "mccraigmccraig@googlemail.com"

lib/sonar_imap_pull_connector.rb

@@ -1,3 +1,4 @@
+$: << File.expand_path("..", __FILE__)
 require 'rubygems'
 require 'sonar_connector'
 require 'net/imap'

lib/sonar_imap_pull_connector/imap_pull_connector.rb

@@ -9,29 +9,39 @@ class ImapPullConnector < Sonar::Connector::Base
       # max uid value is 2**32-1
       MAX_UID = 2**32-1
 
+      DEFAULT_SSL_PORT = 993
+      DEFAULT_PLAIN_PORTDEFAULT_PLAIN_PORT = 143
+
+      DEFAULT_BATCH_SIZE = 100
+
       attr_reader :host
       attr_reader :port
       attr_reader :usessl
       attr_reader :user
       attr_reader :password
       attr_reader :folders
+      attr_reader :batch_size
 
       def parse(settings)
-        ["host", "port", "user", "password", "folders"].each do |param|
+        ["host", "user", "password", "folders"].each do |param|
           raise Sonar::Connector::InvalidConfig.new("#{self.to_s}: param '#{param}' is blank") if settings[param].blank?
         end
 
-        settings.each do |k,v|
-          self.instance_variable_set("@#{k}", v)
-        end
+        @host = settings["host"]
+        @usessl = settings["usessl"] || true
+        @port = settings["port"] || (@usessl ? DEFAULT_SSL_PORT : DEFAULT_PLAIN_PORT)
+        @user = settings["user"]
+        @password = settings["password"]
+        @folders = settings["folders"]
+        @batch_size = settings["batch_size"] || DEFAULT_BATCH_SIZE
       end
 
       def action
         fs = [*folders]
-        log.info "opening connection to : imap://#{user}@#{host}:#{port}/ for folders #{f.inspect}"
+        log.info "opening connection to : imap://#{user}@#{host}:#{port}/ for folders #{fs.inspect}"
 
         imap = Net::IMAP.new(host, port, usessl)
-        imap.login(user, pass)
+        imap.login(user, password)
         log.info "logged in"
 
         state[:folder_last_uids] ||= {}
@@ -43,10 +53,12 @@ def action
           log.info "retrieving from folder: #{f}, uid>=#{next_uid}"
 
           # min uid value is 1
-          uids = imap.search(["UID", "#{next_uid}:#{MAX_UID}"])
+          uids = imap.uid_search(["UID", "#{next_uid}:#{MAX_UID}"])[0...batch_size]
           uids.each do |uid|
+            log.debug "[#{uid}]"
             fetch_and_save(imap, uid)
             state[:folder_last_uids][f] = uid
+            save_state
           end
           log.info "finished folder: #{f}, last_uid=#{state[:folder_last_uids][f]}"
         end
@@ -57,7 +69,19 @@ def action
       def fetch_and_save(imap, msg_uid)
         msg = imap.uid_fetch(msg_uid, "RFC822.HEADER")[0]
         headers = msg.attr["RFC822.HEADER"]
-        working.add("#{headers}\n\n\n\n\n\n", "#{msg_uid}")
+        content = "#{headers}\n\n\n\n\n\n"
+        json = mail_to_json(content, Time.now)
+
+        complete.add(json, "#{msg_uid}")
+      end
+
+      def mail_to_json(content, timestamp)
+        {
+          "rfc822_base64"=>Base64.encode64(content),
+          "name"=>self.name,
+          "retrieved_at"=>timestamp.to_s,
+          "source_info"=>"connector_class: #{self.class}, connector_name: #{self.name}, host: #{self.host}, port: #{self.port}, user: #{self.user}"
+        }.to_json
       end
     end
   end