Code review: 348740043: Replaced timelib by dfdatetime in file entry …

…filter

config/dpkg/changelog

@@ -2,4 +2,4 @@ plaso (20180507-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline <log2timeline-dev@googlegroups.com>  Mon, 07 May 2018 06:21:16 +0200
+ -- Log2Timeline <log2timeline-dev@googlegroups.com>  Mon, 07 May 2018 20:19:26 +0200

config/dpkg/control

@@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline)
 
 Package: python-plaso
 Architecture: all
-Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-binplist (>= 0.1.4), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-construct (>= 2.5.2), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180324), python-dfvfs (>= 20180326), python-dfwinreg (>= 20170521), python-dpkt (>= 1.8), python-efilter (>= 1.5), python-future (>= 0.16.0), python-hachoir-core (>= 1.3.3), python-hachoir-metadata (>= 1.3.3), python-hachoir-parser (>= 1.3.4), python-idna (>= 2.5), python-pefile (>= 2017.5.26), python-psutil (>= 5.4.3), python-pyparsing (>= 2.0.3), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.2.1), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.7.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends}
+Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-binplist (>= 0.1.4), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-construct (>= 2.5.2), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180501), python-dfvfs (>= 20180326), python-dfwinreg (>= 20170521), python-dpkt (>= 1.8), python-efilter (>= 1.5), python-future (>= 0.16.0), python-hachoir-core (>= 1.3.3), python-hachoir-metadata (>= 1.3.3), python-hachoir-parser (>= 1.3.4), python-idna (>= 2.5), python-pefile (>= 2017.5.26), python-psutil (>= 5.4.3), python-pyparsing (>= 2.0.3), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.2.1), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.7.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends}
 Description: Python 2 module of plaso (log2timeline)
  Plaso (log2timeline) is a framework to create super timelines. Its
  purpose is to extract timestamps from various files found on typical

config/travis/runtests.sh

@@ -8,7 +8,11 @@
 # Exit on error.
 set -e;
 
-if test "${TARGET}" = "pylint";
+if test "${TARGET}" = "jenkins";
+then
+	./config/jenkins/linux/run_end_to_end_tests.sh;
+
+elif test "${TARGET}" = "pylint";
 then
 	pylint --version
 

dependencies.ini

@@ -58,7 +58,7 @@ version_property: __version__
 
 [dfdatetime]
 dpkg_name: python-dfdatetime
-minimum_version: 20180324
+minimum_version: 20180501
 rpm_name: python-dfdatetime
 version_property: __version__
 

plaso/dependencies.py

@@ -27,7 +27,7 @@
     'construct': ('__version__', '2.5.2', '2.5.3', True),
     'Crypto': ('__version__', '2.6', None, True),
     'dateutil': ('__version__', '1.5', None, True),
-    'dfdatetime': ('__version__', '20180324', None, True),
+    'dfdatetime': ('__version__', '20180501', None, True),
     'dfvfs': ('__version__', '20180326', None, True),
     'dfwinreg': ('__version__', '20170521', None, True),
     'dpkt': ('__version__', '1.8', None, True),

plaso/filters/file_entry.py

@@ -9,8 +9,9 @@
 
 import pysigscan
 
+from dfdatetime import time_elements
+
 from plaso.lib import py2to3
-from plaso.lib import timelib
 
 
 class FileEntryFilter(object):
@@ -41,10 +42,18 @@ class DateTimeFileEntryFilter(FileEntryFilter):
   """Date and time-based file entry filter."""
 
   _DATE_TIME_RANGE_TUPLE = collections.namedtuple(
-      'date_time_range_tuple', 'time_value start_timestamp end_timestamp')
+      'date_time_range_tuple', 'time_value start_date_time end_date_time')
+
+  # Maps the time value of the date time range to a file entry attribute name.
+  _TIME_VALUE_MAPPINGS = {
+      'atime': 'access_time',
+      'bkup': 'backup_time',
+      'ctime': 'change_time',
+      'crtime': 'creation_time',
+      'dtime': 'deletion_time',
+      'mtime': 'modification_time'}
 
-  _SUPPORTED_TIME_VALUES = frozenset([
-      'atime', 'bkup', 'ctime', 'crtime', 'dtime', 'mtime'])
+  _SUPPORTED_TIME_VALUES = frozenset(_TIME_VALUE_MAPPINGS.keys())
 
   def __init__(self):
     """Initializes a date and time-based file entry filter."""
@@ -79,28 +88,27 @@ def AddDateTimeRange(
 
     time_value_lower = time_value.lower()
     if time_value_lower not in self._SUPPORTED_TIME_VALUES:
-      raise ValueError(
-          'Unsupported time value: {0:s}.'.format(time_value))
+      raise ValueError('Unsupported time value: {0:s}.'.format(time_value))
 
+    start_date_time = None
     if start_time_string:
-      start_timestamp = timelib.Timestamp.CopyFromString(start_time_string)
-    else:
-      start_timestamp = None
+      start_date_time = time_elements.TimeElementsInMicroseconds()
+      start_date_time.CopyFromDateTimeString(start_time_string)
 
+    end_date_time = None
     if end_time_string:
-      end_timestamp = timelib.Timestamp.CopyFromString(end_time_string)
-    else:
-      end_timestamp = None
+      end_date_time = time_elements.TimeElementsInMicroseconds()
+      end_date_time.CopyFromDateTimeString(end_time_string)
 
     # Make sure that the end timestamp occurs after the beginning.
     # If not then we need to reverse the time range.
-    if (None not in [start_timestamp, end_timestamp] and
-        start_timestamp > end_timestamp):
+    if (None not in (start_date_time, end_date_time) and
+        start_date_time > end_date_time):
       raise ValueError(
           'Invalid date time value start must be earlier than end.')
 
     self._date_time_ranges.append(self._DATE_TIME_RANGE_TUPLE(
-        time_value_lower, start_timestamp, end_timestamp))
+        time_value_lower, start_date_time, end_date_time))
 
   def Matches(self, file_entry):
     """Compares the file entry against the filter.
@@ -115,28 +123,22 @@ def Matches(self, file_entry):
     if not self._date_time_ranges:
       return None
 
-    stat_object = file_entry.GetStat()
     for date_time_range in self._date_time_ranges:
-      time_value = date_time_range.time_value
-      timestamp = getattr(stat_object, time_value, None)
-      if timestamp is None:
+      time_attribute = self._TIME_VALUE_MAPPINGS.get(
+          date_time_range.time_value, None)
+      if not time_attribute:
         continue
 
-      nano_time_value = '{0:s}_nano'.format(time_value)
-      nano_time_value = getattr(stat_object, nano_time_value, None)
-
-      timestamp = timelib.Timestamp.FromPosixTime(timestamp)
-      if nano_time_value is not None:
-        # Note that the _nano values are in intervals of 100th nano seconds.
-        nano_time_value, _ = divmod(nano_time_value, 10)
-        timestamp += nano_time_value
+      timestamp = getattr(file_entry, time_attribute, None)
+      if timestamp is None:
+        continue
 
-      if (date_time_range.start_timestamp is not None and
-          timestamp < date_time_range.start_timestamp):
+      if (date_time_range.start_date_time is not None and
+          timestamp < date_time_range.start_date_time):
         return False
 
-      if (date_time_range.end_timestamp is not None and
-          timestamp > date_time_range.end_timestamp):
+      if (date_time_range.end_date_time is not None and
+          timestamp > date_time_range.end_date_time):
         return False
 
     return True
@@ -149,23 +151,21 @@ def Print(self, output_writer):
     """
     if self._date_time_ranges:
       for date_time_range in self._date_time_ranges:
-        if date_time_range.start_timestamp is None:
-          end_time_string = timelib.Timestamp.CopyToIsoFormat(
-              date_time_range.end_timestamp)
+        if date_time_range.start_date_time is None:
+          end_time_string = date_time_range.end_date_time.CopyToDateTimeString()
           output_writer.Write('\t{0:s} after {1:s}\n'.format(
               date_time_range.time_value, end_time_string))
 
-        elif date_time_range.end_timestamp is None:
-          start_time_string = timelib.Timestamp.CopyToIsoFormat(
-              date_time_range.start_timestamp)
+        elif date_time_range.end_date_time is None:
+          start_time_string = (
+              date_time_range.start_date_time.CopyToDateTimeString())
           output_writer.Write('\t{0:s} before {1:s}\n'.format(
               date_time_range.time_value, start_time_string))
 
         else:
-          start_time_string = timelib.Timestamp.CopyToIsoFormat(
-              date_time_range.start_timestamp)
-          end_time_string = timelib.Timestamp.CopyToIsoFormat(
-              date_time_range.end_timestamp)
+          start_time_string = (
+              date_time_range.start_date_time.CopyToDateTimeString())
+          end_time_string = date_time_range.end_date_time.CopyToDateTimeString()
           output_writer.Write('\t{0:s} between {1:s} and {2:s}\n'.format(
               date_time_range.time_value, start_time_string,
               end_time_string))

requirements.txt

@@ -8,7 +8,7 @@ binplist >= 0.1.4
 certifi >= 2016.9.26
 chardet >= 2.0.1
 construct >= 2.5.2,<= 2.5.3
-dfdatetime >= 20180324
+dfdatetime >= 20180501
 dfvfs >= 20180326
 dfwinreg >= 20170521
 dpkt >= 1.8

setup.cfg

@@ -46,7 +46,7 @@ requires = PyYAML >= 3.10
            python-construct >= 2.5.2
            python-crypto >= 2.6
            python-dateutil >= 1.5
-           python-dfdatetime >= 20180324
+           python-dfdatetime >= 20180501
            python-dfvfs >= 20180326
            python-dfwinreg >= 20170521
            python-dpkt >= 1.8

tests/cli/image_export_tool.py

@@ -255,8 +255,8 @@ def testPrintFilterCollection(self):
 
     expected_output = b'\n'.join([
         b'Filters:',
-        (b'\tctime between 2012-05-25T15:59:00+00:00 and '
-         b'2012-05-25T15:59:20+00:00'),
+        (b'\tctime between 2012-05-25 15:59:00.000000 and '
+         b'2012-05-25 15:59:20.000000'),
         b''])
     output = output_writer.ReadOutput()
     self.assertEqual(output, expected_output)

tests/filters/file_entry.py

@@ -138,10 +138,10 @@ def testPrint(self):
     test_filter.Print(output_writer)
 
     expected_output = [
-        (b'\tctime between 2012-05-25T15:59:20+00:00 and '
-         b'2012-05-25T15:59:25+00:00'),
-        b'\tatime after 2012-05-25T15:59:25+00:00',
-        b'\tmtime before 2012-05-25T15:59:20+00:00',
+        (b'\tctime between 2012-05-25 15:59:20.000000 and '
+         b'2012-05-25 15:59:25.000000'),
+        b'\tatime after 2012-05-25 15:59:25.000000',
+        b'\tmtime before 2012-05-25 15:59:20.000000',
         b'']
 
     output = output_writer.ReadOutput()