Code review: 339420043: Updates to support dfdatetime normalized time…

…stamp changes log2timeline#1728

.pylintrc

@@ -7,7 +7,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=pybde,pyesedb,pyevt,pyevtx,pyewf,pyfsntfs,pyfvde,pyfwnt,pyfwsi,pylnk,pymsiecf,pyolecf,pyqcow,pyregf,pyscca,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvshadow,pyvslvm
+extension-pkg-whitelist=pybde,pyesedb,pyevt,pyevtx,pyewf,pyfsntfs,pyfvde,pyfwnt,pyfwsi,pylnk,pymsiecf,pyolecf,pyqcow,pyregf,pyscca,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvshadow,pyvslvm,yara
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.

.travis.yml

@@ -1,4 +1,6 @@
 language: python
+virtualenv:
+  system_site_packages: true
 matrix:
   include:
   - env: TARGET="pylint"
@@ -37,6 +39,7 @@ matrix:
     language: generic
   allow_failures:
   - env: TARGET="linux-python34"
+  - env: [TARGET="linux-python27-tox", TOXENV="py27"]
   - env: [TARGET="linux-python34-tox", TOXENV="py34"]
 install:
 - ./config/travis/install.sh

appveyor.yml

@@ -19,7 +19,7 @@ install:
 - cmd: "%PYTHON%\\Scripts\\pip.exe install pywin32 WMI"
 - cmd: "%PYTHON%\\python.exe %PYTHON%\\Scripts\\pywin32_postinstall.py -install"
 - cmd: git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools
-- cmd: mkdir dependencies && set PYTHONPATH=..\l2tdevtools && "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type x86 --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode binplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dpkt efilter funcsigs future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lzma mock pbr pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python
+- cmd: mkdir dependencies && set PYTHONPATH=..\l2tdevtools && "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type x86 --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode binplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dpkt efilter funcsigs future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python
 
 build: off
 

config/dpkg/changelog

@@ -1,5 +1,5 @@
-plaso (20180324-1) unstable; urgency=low
+plaso (20180325-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline <log2timeline-dev@googlegroups.com>  Sat, 24 Mar 2018 14:48:19 +0100
+ -- Log2Timeline <log2timeline-dev@googlegroups.com>  Sun, 25 Mar 2018 12:02:03 +0200

config/dpkg/control

@@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline)
 
 Package: python-plaso
 Architecture: all
-Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-binplist (>= 0.1.4), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-construct (>= 2.5.2), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180110), python-dfvfs (>= 20171230), python-dfwinreg (>= 20170521), python-dpkt (>= 1.8), python-efilter (>= 1.5), python-future (>= 0.16.0), python-hachoir-core (>= 1.3.3), python-hachoir-metadata (>= 1.3.3), python-hachoir-parser (>= 1.3.4), python-idna (>= 2.5), python-pefile (>= 2017.5.26), python-psutil (>= 1.2.1), python-pyparsing (>= 2.0.3), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.2.1), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.7.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends}
+Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-binplist (>= 0.1.4), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-construct (>= 2.5.2), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180324), python-dfvfs (>= 20180326), python-dfwinreg (>= 20170521), python-dpkt (>= 1.8), python-efilter (>= 1.5), python-future (>= 0.16.0), python-hachoir-core (>= 1.3.3), python-hachoir-metadata (>= 1.3.3), python-hachoir-parser (>= 1.3.4), python-idna (>= 2.5), python-pefile (>= 2017.5.26), python-psutil (>= 1.2.1), python-pyparsing (>= 2.0.3), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.2.1), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.7.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends}
 Description: Python 2 module of plaso (log2timeline)
  Plaso (log2timeline) is a framework to create super timelines. Its
  purpose is to extract timestamps from various files found on typical

config/jenkins/start_slave.py

@@ -1,14 +1,15 @@
 """Script to create Jenkins Slaves."""
 
+from __future__ import print_function
 from __future__ import unicode_literals
 
 import argparse
 import json
 import sys
 import time
 
-from googleapiclient import discovery
-from googleapiclient import errors as apierrors
+from googleapiclient import discovery  # pylint: disable=import-error
+from googleapiclient import errors as apierrors  # pylint: disable=import-error
 
 #pylint: disable=no-member
 
@@ -24,7 +25,6 @@ def __init__(self, project, zone=None):
       project (str): the GCE project name.
       zone (str): the destination GCP zone.
     """
-
     self._project = project
     self._zone = zone
 
@@ -103,10 +103,9 @@ def CreateInstance(
           the instance, in the form {'persistent_disk_name': 'device_name'}.
         scopes (Optional[list[str]]): the list of scopes to set for the instance
     """
-
     scopes = scopes or self.DEFAULT_SCOPES
 
-    print 'Creating new instance {0:s}'.format(instance_name)
+    print('Creating new instance {0:s}'.format(instance_name))
 
     project_url = 'compute/v1/projects/{0:s}'.format(self._project)
     machine_type_url = '{0:s}/zones/{1:s}/machineTypes/{2:s}'.format(
@@ -152,7 +151,12 @@ def CreateInstance(
     self._WaitForOperation(operation)
 
 
-if __name__ == '__main__':
+def Main():
+  """The main function.
+
+  Returns:
+    bool: True if successful or False otherwise.
+  """
   parser = argparse.ArgumentParser()
   parser.add_argument(
       '--attach_persistent_disk', action='append', required=False,
@@ -246,9 +250,18 @@ def CreateInstance(
     status = error_dict['error'].get('code', None)
     error_message = error_dict['error'].get('message', '')
     if status == 409 and error_message.endswith('already exists'):
-      print error_message
+      print(error_message)
     if status == 400 and error_message.endswith(
         'The referenced image resource cannot be found.'):
-      print error_message
+      print(error_message)
     else:
       raise error
+
+  return True
+
+
+if __name__ == '__main__':
+  if not Main():
+    sys.exit(1)
+  else:
+    sys.exit(0)

config/travis/install.sh

@@ -5,7 +5,7 @@
 # This file is generated by l2tdevtools update-dependencies.py any dependency
 # related changes should be made in dependencies.ini.
 
-L2TBINARIES_DEPENDENCIES="PyYAML XlsxWriter artifacts bencode binplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dpkt efilter future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lzma pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
+L2TBINARIES_DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode binplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dpkt efilter future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
 
 L2TBINARIES_TEST_DEPENDENCIES="funcsigs mock pbr";
 

config/travis/runtests.sh

@@ -12,7 +12,7 @@ if test "${TARGET}" = "pylint";
 then
 	pylint --version
 
-	for FILE in `find setup.py plaso tests -name \*.py`;
+	for FILE in `find setup.py config plaso tests tools -name \*.py`;
 	do
 		echo "Checking: ${FILE}";
 

dependencies.ini

@@ -58,13 +58,13 @@ version_property: __version__
 
 [dfdatetime]
 dpkg_name: python-dfdatetime
-minimum_version: 20180110
+minimum_version: 20180324
 rpm_name: python-dfdatetime
 version_property: __version__
 
 [dfvfs]
 dpkg_name: python-dfvfs
-minimum_version: 20171230
+minimum_version: 20180326
 rpm_name: python-dfvfs
 version_property: __version__
 
@@ -123,8 +123,9 @@ rpm_name: python2-idna
 
 [lzma]
 dpkg_name: python-backports.lzma
-pypi_name: backports.lzma
 is_optional: true
+l2tbinaries_name: backports.lzma
+pypi_name: backports.lzma
 python2_only: true
 rpm_name: python-backports-lzma
 version_property: __version__

plaso/__init__.py

@@ -8,4 +8,4 @@
 
 from __future__ import unicode_literals
 
-__version__ = '20180324'
+__version__ = '20180325'

plaso/analysis/interface.py

@@ -497,16 +497,18 @@ def _CheckPythonVersionAndDisableWarnings(self):
           'can allow an attacker to read or modify SSL encrypted data. '
           'Please update. Further SSL warnings will be suppressed. See '
           'https://www.python.org/dev/peps/pep-0466/ for more information.')
+
       # Some distributions de-vendor urllib3 from requests, so we have to
       # check if this has occurred and disable warnings in the correct
       # package.
-      if (hasattr(requests, 'packages') and
-          hasattr(requests.packages, 'urllib3') and
-          hasattr(requests.packages.urllib3, 'disable_warnings')):
-        requests.packages.urllib3.disable_warnings()
-      else:
-        if urllib3 and hasattr(urllib3, 'disable_warnings'):
-          urllib3.disable_warnings()
+      urllib3_module = urllib3
+      if not urllib3_module:
+        if hasattr(requests, 'packages'):
+          urllib3_module = getattr(requests.packages, 'urllib3')
+
+      if urllib3_module and hasattr(urllib3_module, 'disable_warnings'):
+        urllib3_module.disable_warnings()
+
     self._checked_for_old_python_version = True
 
   @abc.abstractmethod

plaso/cli/psort_tool.py

@@ -106,7 +106,7 @@ def __init__(self, input_reader=None, output_writer=None):
     self.list_output_modules = False
     self.list_profilers = False
 
-  def _CheckStorageFile(self, storage_file_path):
+  def _CheckStorageFile(self, storage_file_path):  # pylint: disable=arguments-differ
     """Checks if the storage file path is valid.
 
     Args:

plaso/dependencies.py

@@ -27,8 +27,8 @@
     'construct': ('__version__', '2.5.2', '2.5.3', True),
     'Crypto': ('__version__', '2.6', None, True),
     'dateutil': ('__version__', '1.5', None, True),
-    'dfdatetime': ('__version__', '20180110', None, True),
-    'dfvfs': ('__version__', '20171230', None, True),
+    'dfdatetime': ('__version__', '20180324', None, True),
+    'dfvfs': ('__version__', '20180326', None, True),
     'dfwinreg': ('__version__', '20170521', None, True),
     'dpkt': ('__version__', '1.8', None, True),
     'efilter': ('', '1.5', None, True),
@@ -223,7 +223,7 @@ def _ImportPythonModule(module_name):
   try:
     module_object = list(map(__import__, [module_name]))[0]
   except ImportError:
-    return None
+    return
 
   # If the module name contains dots get the upper most module object.
   if '.' in module_name:

plaso/formatters/file_system.py

@@ -25,7 +25,15 @@ class FileStatEventFormatter(interface.ConditionalEventFormatter):
 
   SOURCE_SHORT = 'FILE'
 
+  # The numeric values are for backwards compatibility with plaso files
+  # generated with older versions of dfvfs.
   _FILE_ENTRY_TYPES = {
+      1: 'device',
+      2: 'directory',
+      3: 'file',
+      4: 'link',
+      5: 'socket',
+      6: 'pipe',
       dfvfs_definitions.FILE_ENTRY_TYPE_DEVICE: 'device',
       dfvfs_definitions.FILE_ENTRY_TYPE_DIRECTORY: 'directory',
       dfvfs_definitions.FILE_ENTRY_TYPE_FILE: 'file',

plaso/multi_processing/plaso_xmlrpc.py

@@ -43,7 +43,7 @@ def CallFunction(self):
       return None
 
     try:
-      return rpc_call()
+      return rpc_call()  # pylint: disable=not-callable
     except (
         expat.ExpatError, SocketServer.socket.error,
         xmlrpclib.Fault) as exception:

plaso/parsers/cookie_plugins/ganalytics.py

@@ -8,7 +8,7 @@
 if sys.version_info[0] < 3:
   import urllib as urlparse
 else:
-  from urllib import parse as urlparse
+  from urllib import parse as urlparse  # pylint: disable=no-name-in-module
 
 # pylint: disable=wrong-import-position
 from dfdatetime import posix_time as dfdatetime_posix_time

plaso/parsers/sqlite_plugins/safari.py

@@ -18,9 +18,9 @@ class SafariHistoryPageVisitedEventData(events.EventData):
   """Safari history event data.
 
   Attributes:
+    host (str): hostname of the server.
     title (str): title of the webpage visited.
     url (str): URL visited.
-    host(str): hostname of the server.
     visit_count (int): number of times the website was visited.
     was_http_non_get (bool): True if the webpage was visited using a
         non-GET HTTP request.
@@ -30,14 +30,14 @@ class SafariHistoryPageVisitedEventData(events.EventData):
 
   def __init__(self):
     """Initializes event data."""
-    super(SafariHistoryPageVisitedEventData,
-          self).__init__(data_type=self.DATA_TYPE)
+    super(SafariHistoryPageVisitedEventData, self).__init__(
+        data_type=self.DATA_TYPE)
+    self.host = None
     self.title = None
     self.url = None
     self.visit_count = None
-    self.host = None
-    self.was_http_non_get = None
     self.visit_redirect_source = None
+    self.was_http_non_get = None
 
 
 class SafariHistoryPluginSqlite(interface.SQLitePlugin):

plaso/parsers/trendmicroav.py

@@ -92,7 +92,7 @@ def _ParseTimestamp(self, parser_mediator, row):
 
     If the Trend Micro log comes from a version that provides a Unix timestamp,
     use that directly; it provides the advantages of UTC and of second
-    precision.  Otherwise fall back onto the local-timezone date and time.
+    precision. Otherwise fall back onto the local-timezone date and time.
 
     Args:
       parser_mediator (ParserMediator): mediates interactions between parsers
@@ -159,7 +159,8 @@ def _ConvertToTimestamp(self, date, time):
     date_time = dfdatetime_time_elements.TimeElements(
         time_elements_tuple=time_elements_tuple)
     date_time.is_local_time = True
-    date_time.precision = dfdatetime_definitions.PRECISION_1_MINUTE
+    # TODO: add functionality to dfdatetime to control precision.
+    date_time._precision = dfdatetime_definitions.PRECISION_1_MINUTE  # pylint: disable=protected-access
 
     return date_time
 

plaso/parsers/winjob.py

@@ -234,7 +234,8 @@ def ParseFileObject(self, parser_mediator, file_object, **kwargs):
           date_time = dfdatetime_time_elements.TimeElements(
               time_elements_tuple=time_elements_tuple)
           date_time.is_local_time = True
-          date_time.precision = dfdatetime_definitions.PRECISION_1_MINUTE
+          # TODO: add functionality to dfdatetime to control precision.
+          date_time._precision = dfdatetime_definitions.PRECISION_1_MINUTE  # pylint: disable=protected-access
         except ValueError:
           date_time = None
           parser_mediator.ProduceExtractionError(
@@ -255,7 +256,8 @@ def ParseFileObject(self, parser_mediator, file_object, **kwargs):
           date_time = dfdatetime_time_elements.TimeElements(
               time_elements_tuple=time_elements_tuple)
           date_time.is_local_time = True
-          date_time.precision = dfdatetime_definitions.PRECISION_1_DAY
+          # TODO: add functionality to dfdatetime to control precision.
+          date_time._precision = dfdatetime_definitions.PRECISION_1_DAY  # pylint: disable=protected-access
         except ValueError:
           date_time = None
           parser_mediator.ProduceExtractionError(

requirements.txt

@@ -8,8 +8,8 @@ binplist >= 0.1.4
 certifi >= 2016.9.26
 chardet >= 2.0.1
 construct >= 2.5.2,<= 2.5.3
-dfdatetime >= 20180110
-dfvfs >= 20171230
+dfdatetime >= 20180324
+dfvfs >= 20180326
 dfwinreg >= 20170521
 dpkt >= 1.8
 efilter == 1-1.5

setup.cfg

@@ -46,8 +46,8 @@ requires = PyYAML >= 3.10
            python-construct >= 2.5.2
            python-crypto >= 2.6
            python-dateutil >= 1.5
-           python-dfdatetime >= 20180110
-           python-dfvfs >= 20171230
+           python-dfdatetime >= 20180324
+           python-dfvfs >= 20180326
            python-dfwinreg >= 20170521
            python-dpkt >= 1.8
            python-efilter >= 1.5

setup.py

@@ -180,17 +180,17 @@ def GetScripts():
 
 
 if version_tuple[0] == 2:
-  encoding = sys.stdin.encoding
+  encoding = sys.stdin.encoding  # pylint: disable=invalid-name
 
   # Note that sys.stdin.encoding can be None.
   if not encoding:
     encoding = locale.getpreferredencoding()
 
   # Make sure the default encoding is set correctly otherwise
   # setup.py sdist will fail to include filenames with Unicode characters.
-  reload(sys)
+  reload(sys)  # pylint: disable=undefined-variable
 
-  sys.setdefaultencoding(encoding)
+  sys.setdefaultencoding(encoding)  # pylint: disable=no-member
 
 
 # Unicode in the description will break python-setuptools, hence