343310043: Added support for lz4 compressed Systemd journal events (l…

…og2timeline#2004)
Onager · Jul 23, 2018 · a1b043d · a1b043d
1 parent e621ba6
commit a1b043d
Show file tree

Hide file tree

Showing 14 changed files with 40 additions and 51 deletions.
diff --git a/appveyor.yml b/appveyor.yml
@@ -22,11 +22,11 @@ install:
 - cmd: if [%TARGET%]==[python27] (
     mkdir dependencies &&
     set PYTHONPATH=..\l2tdevtools &&
-    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type x86 --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py funcsigs future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python )
+    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type x86 --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py funcsigs future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lz4 mock pbr pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python )
 - cmd: if [%TARGET%]==[python36] (
     mkdir dependencies &&
     set PYTHONPATH=..\l2tdevtools &&
-    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type x86 --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py funcsigs future idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pefile psutil pycrypto pyparsing pytsk3 pytz pyzmq requests six urllib3 yara-python )
+    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type x86 --msi-targetdir "%PYTHON%" --track dev PyYAML XlsxWriter artifacts bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py funcsigs future idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lz4 mock pbr pefile psutil pycrypto pyparsing pytsk3 pytz pyzmq requests six urllib3 yara-python )
 
 build: off
 

diff --git a/config/dpkg/control b/config/dpkg/control
@@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline)
 
 Package: python-plaso
 Architecture: all
-Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-biplist (>= 1.0.3), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-construct (>= 2.5.2), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180704), python-dfvfs (>= 20180326), python-dfwinreg (>= 20170521), python-dtfabric (>= 20180707), python-efilter (>= 1.5), python-elasticsearch (>= 5.4.0), python-future (>= 0.16.0), python-hachoir-core (>= 1.3.3), python-hachoir-metadata (>= 1.3.3), python-hachoir-parser (>= 1.3.4), python-idna (>= 2.5), python-pefile (>= 2017.5.26), python-psutil (>= 5.4.3), python-pyparsing (>= 2.0.3), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.2.1), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.7.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends}
+Depends: plaso-data, libbde-python (>= 20140531), libesedb-python (>= 20150409), libevt-python (>= 20120410), libevtx-python (>= 20141112), libewf-python (>= 20131210), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20180117), libfwsi-python (>= 20150606), liblnk-python (>= 20150830), libmsiecf-python (>= 20150314), libolecf-python (>= 20151223), libqcow-python (>= 20131204), libregf-python (>= 20150315), libscca-python (>= 20161031), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-artifacts (>= 20170818), python-backports.lzma, python-bencode, python-biplist (>= 1.0.3), python-certifi (>= 2016.9.26), python-chardet (>= 2.0.1), python-construct (>= 2.5.2), python-crypto (>= 2.6), python-dateutil (>= 1.5), python-dfdatetime (>= 20180606), python-dfvfs (>= 20180326), python-dfwinreg (>= 20170521), python-dtfabric (>= 20180604), python-efilter (>= 1.5), python-elasticsearch (>= 5.4.0), python-future (>= 0.16.0), python-hachoir-core (>= 1.3.3), python-hachoir-metadata (>= 1.3.3), python-hachoir-parser (>= 1.3.4), python-idna (>= 2.5), python-lz4 (>= 0.10.0), python-pefile (>= 2017.5.26), python-psutil (>= 5.4.3), python-pyparsing (>= 2.0.3), python-pysqlite2, python-pytsk3 (>= 20160721), python-requests (>= 2.2.1), python-six (>= 1.1.0), python-tz, python-urllib3 (>= 1.7.1), python-xlsxwriter (>= 0.9.3), python-yaml (>= 3.10), python-yara (>= 3.4.0), python-zmq (>= 2.1.11), ${python:Depends}, ${misc:Depends}
 Description: Python 2 module of plaso (log2timeline)
  Plaso (log2timeline) is a framework to create super timelines. Its
  purpose is to extract timestamps from various files found on typical

diff --git a/config/linux/gift_copr_install.sh b/config/linux/gift_copr_install.sh
@@ -31,6 +31,7 @@ PYTHON2_DEPENDENCIES="PyYAML
                       libvmdk-python
                       libvshadow-python
                       libvslvm-python
+                      lz4
                       python-XlsxWriter
                       python-artifacts
                       python-backports-lzma

diff --git a/config/linux/gift_ppa_install.sh b/config/linux/gift_ppa_install.sh
@@ -50,6 +50,7 @@ PYTHON2_DEPENDENCIES="libbde-python
                       python-hachoir-metadata
                       python-hachoir-parser
                       python-idna
+                      python-lz4
                       python-pefile
                       python-psutil
                       python-pyparsing

diff --git a/config/macos/install.sh b/config/macos/install.sh
@@ -8,7 +8,7 @@
 EXIT_SUCCESS=0;
 EXIT_FAILURE=1;
 
-DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
+DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lz4 pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
 
 SCRIPT_NAME=$(basename $0);
 DEPENDENCIES_ONLY=0;

diff --git a/config/macos/make_dist.sh b/config/macos/make_dist.sh
@@ -8,7 +8,7 @@
 EXIT_SUCCESS=0;
 EXIT_FAILURE=1;
 
-DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
+DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lz4 pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
 
 MACOS_VERSION=`sw_vers -productVersion | awk -F '.' '{print $1 "." $2}'`;
 

diff --git a/config/macos/uninstall.sh b/config/macos/uninstall.sh
@@ -8,7 +8,7 @@
 EXIT_SUCCESS=0;
 EXIT_FAILURE=1;
 
-DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
+DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm lz4 pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
 
 SCRIPT_NAME=`basename $0`;
 DEPENDENCIES_ONLY=0;

diff --git a/config/travis/install.sh b/config/travis/install.sh
@@ -5,15 +5,15 @@
 # This file is generated by l2tdevtools update-dependencies.py any dependency
 # related changes should be made in dependencies.ini.
 
-L2TBINARIES_DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
+L2TBINARIES_DEPENDENCIES="PyYAML XlsxWriter artifacts backports.lzma bencode biplist certifi chardet construct dateutil dfdatetime dfvfs dfwinreg dtfabric efilter elasticsearch-py future hachoir-core hachoir-metadata hachoir-parser idna libbde libesedb libevt libevtx libewf libfsntfs libfvde libfwnt libfwsi liblnk libmsiecf libolecf libqcow libregf libscca libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm python-lz4 pefile psutil pycrypto pyparsing pysqlite pytsk3 pytz pyzmq requests six urllib3 yara-python";
 
 L2TBINARIES_TEST_DEPENDENCIES="funcsigs mock pbr";
 
-PYTHON2_DEPENDENCIES="libbde-python libesedb-python libevt-python libevtx-python libewf-python libfsntfs-python libfvde-python libfwnt-python libfwsi-python liblnk-python libmsiecf-python libolecf-python libqcow-python libregf-python libscca-python libsigscan-python libsmdev-python libsmraw-python libvhdi-python libvmdk-python libvshadow-python libvslvm-python python-artifacts python-backports.lzma python-bencode python-biplist python-certifi python-chardet python-construct python-crypto python-dateutil python-dfdatetime python-dfvfs python-dfwinreg python-dtfabric python-efilter python-elasticsearch python-future python-hachoir-core python-hachoir-metadata python-hachoir-parser python-idna python-pefile python-psutil python-pyparsing python-pysqlite2 python-pytsk3 python-requests python-six python-tz python-urllib3 python-xlsxwriter python-yaml python-yara python-zmq";
+PYTHON2_DEPENDENCIES="libbde-python libesedb-python libevt-python libevtx-python libewf-python libfsntfs-python libfvde-python libfwnt-python libfwsi-python liblnk-python libmsiecf-python libolecf-python libqcow-python libregf-python libscca-python libsigscan-python libsmdev-python libsmraw-python libvhdi-python libvmdk-python libvshadow-python libvslvm-python python-artifacts python-backports.lzma python-bencode python-biplist python-certifi python-chardet python-construct python-crypto python-dateutil python-dfdatetime python-dfvfs python-dfwinreg python-dtfabric python-efilter python-elasticsearch python-future python-hachoir-core python-hachoir-metadata python-hachoir-parser python-idna python-lz4 python-pefile python-psutil python-pyparsing python-pysqlite2 python-pytsk3 python-requests python-six python-tz python-urllib3 python-xlsxwriter python-yaml python-yara python-zmq";
 
 PYTHON2_TEST_DEPENDENCIES="python-coverage python-mock python-tox";
 
-PYTHON3_DEPENDENCIES="libbde-python3 libesedb-python3 libevt-python3 libevtx-python3 libewf-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libfwsi-python3 liblnk-python3 libmsiecf-python3 libolecf-python3 libqcow-python3 libregf-python3 libscca-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-bencode python3-biplist python3-certifi python3-chardet python3-construct python3-crypto python3-dateutil python3-dfdatetime python3-dfvfs python3-dfwinreg python3-dtfabric python3-efilter python3-elasticsearch python3-future python3-idna python3-pefile python3-psutil python3-pyparsing python3-pytsk3 python3-requests python3-six python3-tz python3-urllib3 python3-xlsxwriter python3-yaml python3-yara python3-zmq";
+PYTHON3_DEPENDENCIES="libbde-python3 libesedb-python3 libevt-python3 libevtx-python3 libewf-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libfwsi-python3 liblnk-python3 libmsiecf-python3 libolecf-python3 libqcow-python3 libregf-python3 libscca-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-bencode python3-biplist python3-certifi python3-chardet python3-construct python3-crypto python3-dateutil python3-dfdatetime python3-dfvfs python3-dfwinreg python3-dtfabric python3-efilter python3-elasticsearch python3-future python3-idna python3-lz4 python3-pefile python3-psutil python3-pyparsing python3-pytsk3 python3-requests python3-six python3-tz python3-urllib3 python3-xlsxwriter python3-yaml python3-yara python3-zmq";
 
 PYTHON3_TEST_DEPENDENCIES="python3-mock python3-setuptools python3-tox";
 

diff --git a/dependencies.ini b/dependencies.ini
@@ -129,6 +129,12 @@ dpkg_name: python-idna
 minimum_version: 2.5
 rpm_name: python2-idna
 
+[lz4]
+dpkg_name: python-lz4
+is_optional: true
+minimum_version: 0.10.0
+pypi_name: lz4
+
 [lzma]
 dpkg_name: python-backports.lzma
 is_optional: true

diff --git a/plaso/dependencies.py b/plaso/dependencies.py
@@ -38,6 +38,7 @@
     'hachoir_metadata': ('__version__', '1.3.3', None, False),
     'hachoir_parser': ('__version__', '1.3.4', None, False),
     'idna': ('', '2.5', None, True),
+    'lz4': ('', '0.10.0', None, False),
     'lzma': ('__version__', '', None, False),
     'pefile': ('__version__', '2017.5.26', None, True),
     'psutil': ('__version__', '5.4.3', None, True),

diff --git a/plaso/parsers/systemd_journal.py b/plaso/parsers/systemd_journal.py
@@ -6,6 +6,7 @@
 import os
 
 import construct
+import lz4.block
 
 from dfdatetime import posix_time as dfdatetime_posix_time
 
@@ -43,15 +44,6 @@ def __init__(self):
     self.reporter = None
 
 
-# TODO: remove once #2004 is pushed
-class SystemdDirtyJournalEventData(SystemdJournalEventData):
-  """Systemd 'dirty' journal event data.
-
-  Generated when a decompression error was encountered.
-  """
-  DATA_TYPE = 'systemd:journal:dirty'
-
-
 class SystemdJournalParser(interface.FileObjectParser):
   """Parses Systemd Journal files."""
 
@@ -200,7 +192,7 @@ def _ParseItem(self, file_object, offset):
       offset (int): offset to the DATA object.
 
     Returns:
-      tuple[str, str, bool]: key and value of this item, with a 'dirty' bit.
+      tuple[str, str]: key and value of this item.
 
     Raises:
       ParseError: When an unexpected object type is parsed.
@@ -213,23 +205,18 @@ def _ParseItem(self, file_object, offset):
           'Expected an object of type DATA, but got {0:s}'.format(
               object_header.type))
 
-    is_dirty = False
     event_data = file_object.read(payload_size - self._DATA_OBJECT_SIZE)
     if object_header.flags & self._OBJECT_COMPRESSED_FLAG_XZ:
       event_data = lzma.decompress(event_data)
       event_string = event_data.decode('utf-8')
     elif object_header.flags & self._OBJECT_COMPRESSED_FLAG_LZ4:
-      # TODO: implement proper LZ4 decompression (see PR #2004)
-      is_dirty = True
-      event_string = event_data.decode('utf-8', 'ignore')
-      pos = event_string.index('MESSAGE=')
-      if pos >= 0:
-        event_string = 'MESSAGE=' + event_string[pos+8:]
-    else:
-      event_string = event_data.decode('utf-8')
+      decompressed_size = construct.ULInt32('size').parse(event_data[0:7])
+      event_data = lz4.block.decompress(
+          event_data[8:], uncompressed_size=decompressed_size)
 
+    event_string = event_data.decode('utf-8')
     event_key, event_value = event_string.split('=', 1)
-    return event_key, event_value, is_dirty
+    return (event_key, event_value)
 
   def _ParseJournalEntry(self, parser_mediator, file_object, offset):
     """Parses a Systemd journal ENTRY object.
@@ -255,17 +242,12 @@ def _ParseJournalEntry(self, parser_mediator, file_object, offset):
               object_header.type))
 
     fields = {}
-    dirty = False
     for item in entry_object.object_items:
       if item.object_offset < self._max_journal_file_offset:
         raise errors.ParseError(
             'object offset should be after hash tables ({0:d} < {1:d})'.format(
                 offset, self._max_journal_file_offset))
-      # TODO: remove the dirty variable once #2004 is pushed
-      key, value, _dirty = self._ParseItem(file_object, item.object_offset)
-      # We parse a lot of items for one event, we need to remember if one of the
-      # parsed item was dirty.
-      dirty = dirty or _dirty
+      key, value = self._ParseItem(file_object, item.object_offset)
       fields[key] = value
 
     reporter = fields.get('SYSLOG_IDENTIFIER', None)
@@ -274,11 +256,7 @@ def _ParseJournalEntry(self, parser_mediator, file_object, offset):
     else:
       pid = None
 
-    # TODO: remove the dirty variable once #2004 is pushed
-    if dirty:
-      event_data = SystemdDirtyJournalEventData()
-    else:
-      event_data = SystemdJournalEventData()
+    event_data = SystemdJournalEventData()
 
     event_data.body = fields['MESSAGE']
     event_data.hostname = fields['_HOSTNAME']
@@ -299,7 +277,7 @@ def _ParseEntries(self, file_object, offset):
       offset (int): offset of the ENTRY_ARRAY object.
 
     Returns:
-      list[dict]: every ENTRY objects offsets.
+      list[int]: every ENTRY objects offsets.
 
     Raises:
       ParseError: When an unexpected object type is parsed.
@@ -313,8 +291,8 @@ def _ParseEntries(self, file_object, offset):
               object_header.type))
 
     next_array_offset = self._ULInt64.parse_stream(file_object)
-    entry_offests_numbers, _ = divmod((payload_size - 8), 8)
-    for entry_offset in range(entry_offests_numbers):
+    entry_offsets_numbers, _ = divmod((payload_size - 8), 8)
+    for entry_offset in range(entry_offsets_numbers):
       entry_offset = self._ULInt64.parse_stream(file_object)
       if entry_offset != 0:
         entry_offsets.append(entry_offset)

diff --git a/requirements.txt b/requirements.txt
@@ -41,6 +41,7 @@ libvhdi-python >= 20131210
 libvmdk-python >= 20140421
 libvshadow-python >= 20160109
 libvslvm-python >= 20160109
+lz4 >= 0.10.0
 pefile >= 2017.5.26
 psutil >= 5.4.3
 pycrypto >= 2.6