Permalink
Browse files

Add JSON parser for combining different log types. Refactor name=>type.

  • Loading branch information...
1 parent c9b8fe6 commit 2dd00354245fd8ca280eb0122917ea2ef36cd1fb @jamesu committed Jan 22, 2011
View
@@ -16,6 +16,7 @@
require 'parsers/colloquy_parser'
require 'parsers/ircii_parser'
require 'parsers/talker_parser'
+require 'parsers/json_parser'
require 'html_dumper'
View
@@ -16,7 +16,7 @@ def initialize(entry, sender=nil)
@occurred = entry[:occurred]
@content = entry[:content] || ''
@source = entry[:source]
- @type = entry[:name]
+ @type = entry[:type]
end
def is_action
@@ -25,7 +25,7 @@ def is_action
def to_loghash(opts={})
{
- :sender => @sender.to_loghash,
+ :sender => @sender.ident||@sender.name,
:occurred => @occurred,
:content => @content,
:source => @source,
View
@@ -43,20 +43,20 @@ def parse_line(content)
content = words[2..-1].join(' ')
base = {
- :name => :event,
+ :type => :event,
:sender => extract_sender(words[1]),
:occurred => time,
:content => content}
if content.match(/has quit/)
- base[:name] = :userDisconnected
+ base[:type] = :userDisconnected
elsif content.match(/has joined/)
- base[:name] = :userAvailable
+ base[:type] = :userAvailable
elsif content.match(/is now known as (.*)/)
#user = $1.match(USER_MATCH)
- base[:name] = :userNewNickname
+ base[:type] = :userNewNickname
elsif content.match(/has left/)
- base[:name] = :userLeft
+ base[:type] = :userLeft
end
base
@@ -75,7 +75,7 @@ def parse_line(content)
end
return {
- :name => is_action ? :action : :message,
+ :type => is_action ? :action : :message,
:sender => sender,
:occurred => time,
:content => cnt}
@@ -80,7 +80,7 @@ def parse_node(entry)
content = "???"
end
- {:name => entry_type,
+ {:type => entry_type,
:sender => sender,
:source => nil,
:occurred => time,
@@ -66,24 +66,24 @@ def parse_line(content)
if user_match
# i.e. Foo [...] has done something
base = {
- :name => :event,
+ :type => :event,
:sender => {:ident => user_match[2], :name => user_match[1], :host => user_match[3]},
:occurred => time,
:content => user_match[4] }
if user_match[4].match(/has quit/)
- base[:name] = :userDisconnected
+ base[:type] = :userDisconnected
elsif user_match[4].match(/has joined/)
- base[:name] = :userAvailable
+ base[:type] = :userAvailable
elsif user_match[4].match(/has left/)
- base[:name] = :userLeft
+ base[:type] = :userLeft
end
base
else
# i.e. ???
return {
- :name => :event,
+ :type => :event,
:sender => {:ident => words[0], :name => words[0], :host => nil},
:occurred => time,
:content => words[1..-1].join(' ')}
@@ -92,7 +92,7 @@ def parse_line(content)
# User chat / action
chat = extract_message(match[3])
return {
- :name => chat[2] ? :action : :message,
+ :type => chat[2] ? :action : :message,
:sender => chat[0],
:occurred => time,
:content => chat[1]}
@@ -0,0 +1,24 @@
+class JSONParser < Parser
+
+ def initialize(collection)
+ super(collection)
+ end
+
+ def parse(input)
+ data = ActiveSupport::JSON.decode(input.read)
+ data['senders'].each do |sender|
+ @collection.senders[sender['ident']] = Sender.new(sender['ident'],
+ sender['name'],
+ sender['hostmask'])
+ end
+
+ data['entries'].each do |entry|
+ @collection.entries << LogEntry.new({
+ :occurred => Time.parse(entry['occurred']),
+ :content => entry['content'],
+ :source => entry['source'],
+ :type => entry['type'].to_sym
+ }, @collection.senders[entry['sender']])
+ end
+ end
+end
@@ -31,15 +31,15 @@ def parse_event(event)
user = event['user']
base = {
:occurred => Time.at(event['time'].to_i),
- :name => EVENTYPE_MAP[event['type']]||:event,
+ :type => EVENTYPE_MAP[event['type']]||:event,
:source => "talker://#{room['name']}",
:content => event['content'],
:sender => {:ident => user['id'], :name => user['name']}
}
- if base[:name] == :userAvailable
+ if base[:type] == :userAvailable
base[:content] = "left"
- elsif base[:name] == :userDisconnected
+ elsif base[:type] == :userDisconnected
base[:content] = "joined"
end
View
@@ -13,7 +13,7 @@ def merge(sender)
def to_loghash
{
- :ident => @ident,
+ :ident => @ident||@name,
:name => @name,
:hostmask => @hostmask
}
View
@@ -18,15 +18,15 @@
it "should accept messages" do
event = @parser.parse_line("01-04-2010 11:41:28 < woot!~47e0bd37@someclient1: hello folks")
- event[:name].should == :message
+ event[:type].should == :message
event[:content].should == "hello folks"
event[:occurred].to_s.should == "Thu Apr 01 11:41:28 UTC 2010"
event[:sender][:ident].should == "~47e0bd37"
event[:sender][:host].should == "someclient1"
event[:sender][:name].should == "woot"
event = @parser.parse_line("01-04-2010 12:26:44 > Myself: haha")
- event[:name].should == :message
+ event[:type].should == :message
event[:content].should == "haha"
event[:occurred].to_s.should == "Thu Apr 01 12:26:44 UTC 2010"
event[:sender][:ident].should == "Myself"
@@ -36,15 +36,15 @@
it "should accept quit messages" do
event = @parser.parse_line("01-04-2010 00:04:49 -!- LongWind!~long@some.other.server has quit [Quit: Leaving]")
- event[:name].should == :userDisconnected
+ event[:type].should == :userDisconnected
event[:content].should == "has quit [Quit: Leaving]"
event[:occurred].to_s.should == "Thu Apr 01 00:04:49 UTC 2010"
event[:sender][:ident].should == "~long"
event[:sender][:host].should == "some.other.server"
event[:sender][:name].should == "LongWind"
event = @parser.parse_line("01-04-2010 11:08:12 -!- SoGot!|SoGot!~chatzilla@some.server.net has quit [Client closed connection]")
- event[:name].should == :userDisconnected
+ event[:type].should == :userDisconnected
event[:content].should == "has quit [Client closed connection]"
event[:occurred].to_s.should == "Thu Apr 01 11:08:12 UTC 2010"
event[:sender][:ident].should == "|SoGot!~chatzilla"
@@ -54,7 +54,7 @@
it "should accept join messages" do
event = @parser.parse_line("01-04-2010 01:14:48 -!- wooter!~wooter@pool-party.net has joined #letstalk")
- event[:name].should == :userAvailable
+ event[:type].should == :userAvailable
event[:content].should == "has joined #letstalk"
event[:occurred].to_s.should == "Thu Apr 01 01:14:48 UTC 2010"
event[:sender][:ident].should == "~wooter"
@@ -23,7 +23,7 @@
it "should accept quit messages" do
doc = Nokogiri::XML.parse('<event id="VI5I5ZUHY2" name="userDisconnected" occurred="2010-09-02 12:49:53 +0100"><message>JohnSmith disconnected from the server.</message></event>')
event = @parser.parse_node(doc.children[0])
- event[:name].should == :userDisconnected
+ event[:type].should == :userDisconnected
event[:content].should == "JohnSmith disconnected from the server."
event[:occurred].to_s.should == "Thu Sep 02 12:49:53 +0100 2010"
event[:sender].should == nil
@@ -32,7 +32,7 @@
it "should accept join messages" do
doc = Nokogiri::XML.parse('<event id="D1U7LH65IY2" name="userAvailable" occurred="2010-09-02 16:30:17 +0100"><message><span class="member">JohnSmith</span> is now available.</message></event>')
event = @parser.parse_node(doc.children[0])
- event[:name].should == :userAvailable
+ event[:type].should == :userAvailable
event[:content].should == "JohnSmith is now available."
event[:occurred].to_s.should == "Thu Sep 02 16:30:17 +0100 2010"
event[:sender].should == nil
@@ -0,0 +1,93 @@
+{
+ "entries": [
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:36:30 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "johnsmith",
+ "content": "what?"
+ },
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:36:35 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "johnsmith",
+ "content": "you have no chunky bacon?"
+ },
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:36:44 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "myself",
+ "content": "Err..."
+ },
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:36:48 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "johnsmith",
+ "content": "we need some chunky bacon"
+ },
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:36:51 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "johnsmith",
+ "content": "preferrably with extra chunkyness"
+ },
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:36:51 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "myself",
+ "content": "sure, i have loads!"
+ },
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:37:13 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "johnsmith",
+ "content": "great :)"
+ },
+ {
+ "type": "message",
+ "occurred": "Thu Sep 02 10:37:16 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "myself",
+ "content": "ok, well if thats it i will get it ready!"
+ },
+ {
+ "type": "userDisconnected",
+ "occurred": "Thu Sep 02 12:49:53 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "myself",
+ "content": "JohnSmith disconnected from the server."
+ },
+ {
+ "type": "userAvailable",
+ "occurred": "Thu Sep 02 16:30:17 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "myself",
+ "content": "JohnSmith is now available."
+ },
+ {
+ "type": "userDisconnected",
+ "occurred": "Thu Sep 02 17:40:53 +0100 2010",
+ "source": "irc://irc.fudgeyes.net/JohnSmith",
+ "sender": "myself",
+ "content": "JohnSmith disconnected from the server."
+ }
+ ],
+ "senders": [
+ {
+ "hostmask": "~johnsmith@somewhere",
+ "name": "JohnSmith",
+ "ident": "johnsmith"
+ },
+ {
+ "hostmask": "myself@localhost",
+ "name": "myself",
+ "ident": "myself"
+ }
+ ]
+}
View
@@ -22,15 +22,15 @@
event = @parser.parse_line(LOG_OPEN)
event.should == nil
event = @parser.parse_line("16:19 < Myself> yo woot")
- event[:name].should == :message
+ event[:type].should == :message
event[:content].should == "yo woot"
event[:occurred].to_s.should == "Wed Jan 06 16:19:00 UTC 2010"
event[:sender][:ident].should == "Myself"
event[:sender][:host].should == nil
event[:sender][:name].should == "Myself"
event = @parser.parse_line("16:23 < woot> lol why is that? what did you expect?")
- event[:name].should == :message
+ event[:type].should == :message
event[:content].should == "lol why is that? what did you expect?"
event[:occurred].to_s.should == "Wed Jan 06 16:23:00 UTC 2010"
event[:sender][:ident].should == "woot"
@@ -55,7 +55,7 @@
it "should accept quit messages" do
event = @parser.parse_line(LOG_OPEN)
event = @parser.parse_line("15:01 -!- Sick [~Sick@192.168.0.1] has quit [Ping timeout]")
- event[:name].should == :userDisconnected
+ event[:type].should == :userDisconnected
event[:content].should == "has quit [Ping timeout]"
event[:occurred].to_s.should == "Wed Jan 06 15:01:00 UTC 2010"
event[:sender][:ident].should == "~Sick"
@@ -66,7 +66,7 @@
it "should accept join messages" do
event = @parser.parse_line(LOG_OPEN)
event = @parser.parse_line("15:16 -!- Sick [~Sick@192.168.0.1] has joined #letstalk")
- event[:name].should == :userAvailable
+ event[:type].should == :userAvailable
event[:content].should == "has joined #letstalk"
event[:occurred].to_s.should == "Wed Jan 06 15:16:00 UTC 2010"
event[:sender][:ident].should == "~Sick"
View
@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe JSONParser do
+ before do
+ @collection = LogCollection.new
+ @parser = JSONParser.new(@collection)
+ end
+
+ it "should parse messages" do
+ json_data = File.open("tests/fixtures/sample.json") {|f| f.read}
+ @parser.parse(StringIO.new(json_data))
+
+ @collection.entries.length.should == 11
+ @collection.senders.length.should == 2
+ @collection.senders['johnsmith'].should_not == nil
+ @collection.senders['myself'].should_not == nil
+ end
+end
@@ -28,7 +28,7 @@
event = '{ "time": 1270196596, "id": "e93a77b0205e012de69812313d01d943", "type": "leave", "room": { "name": "Main", "id":
819 }, "user": { "name": "fred", "id": 2657, "email": "fred@someserver.com" } }'
event = @parser.parse_event(ActiveSupport::JSON.decode(event))
- event[:name].should == :userDisconnected
+ event[:type].should == :userDisconnected
event[:source].should == "talker://Main"
event[:content].should == "joined"
event[:occurred].to_s.should == "Fri Apr 02 09:23:16 +0100 2010"
@@ -39,7 +39,7 @@
event = '{ "time": 1270196430, "id": "867baae0205e012de69812313d01d943", "type": "join", "room": { "name": "Main", "id": 819
}, "user": { "name": "frodo", "id": 2657, "email": "frodo@someserver.com" } }'
event = @parser.parse_event(ActiveSupport::JSON.decode(event))
- event[:name].should == :userAvailable
+ event[:type].should == :userAvailable
event[:source].should == "talker://Main"
event[:content].should == "left"
event[:occurred].to_s.should == "Fri Apr 02 09:20:30 +0100 2010"
Oops, something went wrong.

0 comments on commit 2dd0035

Please sign in to comment.