Improved compatibility of sqlite parser and plugins with Python 3 log…

…2timeline#1952 (log2timeline#1977)

plaso/parsers/cookie_plugins/ganalytics.py

@@ -3,12 +3,7 @@
 
 from __future__ import unicode_literals
 
-import sys
-
-if sys.version_info[0] < 3:
-  import urllib as urlparse
-else:
-  from urllib import parse as urlparse  # pylint: disable=no-name-in-module
+import codecs
 
 # pylint: disable=wrong-import-position
 from dfdatetime import posix_time as dfdatetime_posix_time
@@ -17,9 +12,14 @@
 from plaso.containers import events
 from plaso.containers import time_events
 from plaso.lib import definitions
+from plaso.lib import py2to3
 from plaso.parsers.cookie_plugins import interface
 from plaso.parsers.cookie_plugins import manager
 
+if py2to3.PY_2:
+  import urllib as urlparse
+else:
+  from urllib import parse as urlparse  # pylint: disable=no-name-in-module
 
 # TODO: determine if __utmc always 0?
 
@@ -89,7 +89,7 @@ def GetEntries(
 
     Args:
       parser_mediator (ParserMediator): parser mediator.
-      cookie_data (bytes): cookie data.
+      cookie_data (str): cookie data.
       url (str): URL or path where the cookie got set.
     """
     fields = cookie_data.split('.')
@@ -355,7 +355,7 @@ def GetEntries(
 
     Args:
       parser_mediator (ParserMediator): parser mediator.
-      cookie_data (bytes): cookie data.
+      cookie_data (str): cookie data.
       url (str): URL or path where the cookie got set.
     """
     fields = cookie_data.split('.')
@@ -410,29 +410,30 @@ def GetEntries(
       for variable in extra_variables:
         key, _, value = variable.partition('=')
 
-        # Cookies can have a variety of different encodings, usually ASCII or
-        # UTF-8, and values may additionally be URL encoded. urllib only
-        # correctly url-decodes ASCII strings, so we'll convert our string
-        # to ASCII first.
-        try:
-          ascii_value = value.encode('ascii')
-        except UnicodeEncodeError:
-          ascii_value = value.encode('ascii', errors='replace')
-          parser_mediator.ProduceExtractionError(
-              'Cookie contains non 7-bit ASCII characters, which have been '
-              'replaced with a "?".')
-
-        utf_stream = urlparse.unquote(ascii_value)
-
-        try:
-          value_line = utf_stream.decode('utf-8')
-        except UnicodeDecodeError:
-          value_line = utf_stream.decode('utf-8', errors='replace')
-          parser_mediator.ProduceExtractionError(
-              'Cookie value did not decode to Unicode string. Non UTF-8 '
-              'characters have been replaced.')
-
-        extra_attributes[key] = value_line
+        # Urllib2 in Python 2 requires a 'str' argument, not 'unicode'. We thus
+        # need to convert the value argument to 'str" and back again, but only
+        # in Python 2.
+        if isinstance(value, py2to3.UNICODE_TYPE) and py2to3.PY_2:
+          try:
+            value = codecs.decode(value, 'ascii')
+          except UnicodeEncodeError:
+            value = codecs.decode(value, 'ascii', errors='replace')
+            parser_mediator.ProduceExtractionError(
+                'Cookie contains non 7-bit ASCII characters, which have been '
+                'replaced with a "?".')
+
+        value = urlparse.unquote(value)
+
+        if py2to3.PY_2:
+          try:
+            value = codecs.encode(value, 'utf-8')
+          except UnicodeDecodeError:
+            value = codecs.encode(value, 'utf-8', errors='replace')
+            parser_mediator.ProduceExtractionError(
+                'Cookie value did not contain a Unicode string. Non UTF-8 '
+                'characters have been replaced.')
+
+        extra_attributes[key] = value
 
     if last_visit_posix_time is not None:
       date_time = dfdatetime_posix_time.PosixTime(

plaso/parsers/sqlite_plugins/chrome_cookies.py

@@ -20,15 +20,15 @@ class ChromeCookieEventData(events.EventData):
 
   Attributes:
     cookie_name (str): name of the cookie.
-    hostname (str): hostname of host that set the cookie value.
+    host (str): hostname of host that set the cookie value.
     httponly (bool): True if the cookie cannot be accessed through client
         side script.
     path (str): path where the cookie got set.
     persistent (bool): True if the cookie is persistent.
     secure (bool): True if the cookie should only be transmitted over a
         secure channel.
     url (str): URL or path where the cookie got set.
-    value (str): value of the cookie.
+    data (str): value of the cookie.
   """
 
   DATA_TYPE = 'chrome:cookie:entry'

plaso/parsers/sqlite_plugins/firefox_cookies.py

@@ -24,7 +24,7 @@ class FirefoxCookieEventData(events.EventData):
     data (str): cookie data.
     httponly (bool): True if the cookie cannot be accessed through client
         side script.
-    hostname (str): hostname of host that set the cookie value.
+    host (str): hostname of host that set the cookie value.
     path (str): URI of the page that set the cookie.
     secure (bool): True if the cookie should only be transmitted over a secure
         channel.

plaso/parsers/sqlite_plugins/skype.py

@@ -55,7 +55,7 @@ def __init__(self):
 
 
 class SkypeSMSEventData(events.EventData):
-  """SKype SMS event data.
+  """Skype SMS event data.
 
   Attributes:
     number (str): phone number where the SMS was sent.
@@ -103,7 +103,7 @@ class SkypeTransferFileEventData(events.EventData):
     destination (str): account that received the file.
     source (str): account that sent the file.
     transferred_filename (str): name of the file transferred.
-    transferred_file_path (str): path of the file transferred.
+    transferred_filepath (str): path of the file transferred.
     transferred_filesize (int): size of the file transferred.
   """
 
@@ -446,7 +446,10 @@ def ParseAccountInformation(
 
     display_name = self._GetRowValue(query_hash, row, 'given_displayname')
     fullname = self._GetRowValue(query_hash, row, 'fullname')
-    username = '{0:s} <{1:s}>'.format(fullname, display_name)
+
+    # TODO: Move this to the formatter, and ensure username is rendered
+    # properly when fullname and/or display_name is None.
+    username = '{0!s} <{1!s}>'.format(fullname, display_name)
 
     event_data = SkypeAccountEventData()
     event_data.country = self._GetRowValue(query_hash, row, 'country')

tests/parsers/sqlite.py

@@ -2,9 +2,9 @@
 # -*- coding: utf-8 -*-
 """Tests for the SQLite database parser."""
 
-import sys
 import unittest
 
+from plaso.lib import py2to3
 from plaso.parsers import sqlite
 # Register all plugins.
 from plaso.parsers import sqlite_plugins  # pylint: disable=unused-import
@@ -57,23 +57,23 @@ def testQueryDatabaseWithWAL(self):
       # Also, Field3 needs to be converted to a string if Python 2 is used
       # because it is a read-write buffer.
       field3 = row['Field3']
-      if sys.version_info[0] < 3:
+      if py2to3.PY_2 and field3 is not None:
         field3 = str(field3)
       row_results.append((
           row['Field1'], row['Field2'], field3))
 
     expected_results = [
-        ('Committed Text 1', 1, b'None'),
-        ('Committed Text 2', 2, b'None'),
-        ('Modified Committed Text 3', 4, b'None'),
-        ('Committed Text 4', 5, b'None'),
-        ('Committed Text 5', 7, b'None'),
-        ('Committed Text 6', 8, b'None'),
-        ('Committed Text 7', 9, b'None'),
+        ('Committed Text 1', 1, None),
+        ('Committed Text 2', 2, None),
+        ('Modified Committed Text 3', 4, None),
+        ('Committed Text 4', 5, None),
+        ('Committed Text 5', 7, None),
+        ('Committed Text 6', 8, None),
+        ('Committed Text 7', 9, None),
         ('Unhashable Row 1', 10, b'Binary Text!\x01\x02\x03'),
         ('Unhashable Row 2', 11, b'More Binary Text!\x01\x02\x03'),
-        ('New Text 1', 12, b'None'),
-        ('New Text 2', 13, b'None')]
+        ('New Text 1', 12, None),
+        ('New Text 2', 13, None)]
 
     self.assertEqual(expected_results, row_results)
 
@@ -93,21 +93,21 @@ def testQueryDatabaseWithoutWAL(self):
       # Also, Field3 needs to be converted to a string if Python 2 is used
       # because it is a read-write buffer.
       field3 = row['Field3']
-      if sys.version_info[0] < 3:
+      if py2to3.PY_2 and field3:
         field3 = str(field3)
       row_results.append((
           row['Field1'], row['Field2'], field3))
 
     expected_results = [
-        ('Committed Text 1', 1, b'None'),
-        ('Committed Text 2', 2, b'None'),
-        ('Deleted Text 1', 3, b'None'),
-        ('Committed Text 3', 4, b'None'),
-        ('Committed Text 4', 5, b'None'),
-        ('Deleted Text 2', 6, b'None'),
-        ('Committed Text 5', 7, b'None'),
-        ('Committed Text 6', 8, b'None'),
-        ('Committed Text 7', 9, b'None'),
+        ('Committed Text 1', 1, None),
+        ('Committed Text 2', 2, None),
+        ('Deleted Text 1', 3, None),
+        ('Committed Text 3', 4, None),
+        ('Committed Text 4', 5, None),
+        ('Deleted Text 2', 6, None),
+        ('Committed Text 5', 7, None),
+        ('Committed Text 6', 8, None),
+        ('Committed Text 7', 9, None),
         ('Unhashable Row 1', 10, b'Binary Text!\x01\x02\x03')]
 
     self.assertEqual(expected_results, row_results)

tests/parsers/sqlite_plugins/interface.py

@@ -4,12 +4,12 @@
 
 from __future__ import unicode_literals
 
-import sys
 import unittest
 
 from plaso.containers import events
 from plaso.containers import time_events
 from plaso.lib import definitions
+from plaso.lib import py2to3
 from plaso.lib import timelib
 from plaso.parsers.sqlite_plugins import interface
 
@@ -78,7 +78,7 @@ def ParseMyTableRow(self, parser_mediator, query, row, **unused_kwargs):
 
     # If Python 2 is used field3 needs to be converted to a string
     # because it is a read-write buffer.
-    if sys.version_info[0] < 3:
+    if py2to3.PY_2 and field3 is not None:
       field3 = str(field3)
 
     self.results.append((field1, field2, field3))
@@ -109,17 +109,17 @@ def testProcessWithWAL(self):
         ['wal_database.db'], plugin, wal_path=wal_file)
 
     expected_results = [
-        ('Committed Text 1', 1, b'None'),
-        ('Committed Text 2', 2, b'None'),
-        ('Modified Committed Text 3', 4, b'None'),
-        ('Committed Text 4', 5, b'None'),
-        ('Committed Text 5', 7, b'None'),
-        ('Committed Text 6', 8, b'None'),
-        ('Committed Text 7', 9, b'None'),
+        ('Committed Text 1', 1, None),
+        ('Committed Text 2', 2, None),
+        ('Modified Committed Text 3', 4, None),
+        ('Committed Text 4', 5, None),
+        ('Committed Text 5', 7, None),
+        ('Committed Text 6', 8, None),
+        ('Committed Text 7', 9, None),
         ('Unhashable Row 1', 10, b'Binary Text!\x01\x02\x03'),
         ('Unhashable Row 2', 11, b'More Binary Text!\x01\x02\x03'),
-        ('New Text 1', 12, b'None'),
-        ('New Text 2', 13, b'None')]
+        ('New Text 1', 12, None),
+        ('New Text 2', 13, None)]
 
     self.assertEqual(plugin.results, expected_results)
 
@@ -130,15 +130,15 @@ def testProcessWithoutWAL(self):
     self._ParseDatabaseFileWithPlugin(['wal_database.db'], plugin)
 
     expected_results = [
-        ('Committed Text 1', 1, b'None'),
-        ('Committed Text 2', 2, b'None'),
-        ('Deleted Text 1', 3, b'None'),
-        ('Committed Text 3', 4, b'None'),
-        ('Committed Text 4', 5, b'None'),
-        ('Deleted Text 2', 6, b'None'),
-        ('Committed Text 5', 7, b'None'),
-        ('Committed Text 6', 8, b'None'),
-        ('Committed Text 7', 9, b'None'),
+        ('Committed Text 1', 1, None),
+        ('Committed Text 2', 2, None),
+        ('Deleted Text 1', 3, None),
+        ('Committed Text 3', 4, None),
+        ('Committed Text 4', 5, None),
+        ('Deleted Text 2', 6, None),
+        ('Committed Text 5', 7, None),
+        ('Committed Text 6', 8, None),
+        ('Committed Text 7', 9, None),
         ('Unhashable Row 1', 10, b'Binary Text!\x01\x02\x03')]
 
     self.assertEqual(plugin.results, expected_results)

tests/parsers/sqlite_plugins/skype.py

@@ -21,9 +21,10 @@ def testProcess(self):
     """Tests the Process function on a Skype History database file.
 
       The History file contains 24 events:
-      4 call events
+      3 call events
       4 transfers file events
-      1 sms events
+      1 sms event
+      1 account event
       15 chat events
 
       Events used:
@@ -45,6 +46,7 @@ def testProcess(self):
     number_of_files = 0
     number_of_sms = 0
     number_of_chats = 0
+    number_of_account_events = 0
     for event in events:
       if event.data_type == 'skype:event:call':
         number_of_calls += 1
@@ -54,6 +56,8 @@ def testProcess(self):
         number_of_sms += 1
       if event.data_type == 'skype:event:chat':
         number_of_chats += 1
+      if event.data_type == 'skype:event:account':
+        number_of_account_events += 1
 
     self.assertEqual(number_of_files, 4)
     self.assertEqual(number_of_sms, 1)