Upload Windows Telemetry ETW Monitor

files/wintel_etwmonitor/tiv/README.md

@@ -0,0 +1,78 @@
+# Windows Telemetry ETW Monitor: Telemetry Information Visualization (TIV) Framework
+
+This folder ("tiv") stores the implementation of the Telemetry Information Visualization (TIV) framework, part of the Windows Telemetry ETW Monitor. The TIV framework is a set of Python scripts that visualize information and statistics based on the data produced by the Windbg Framework (also part of the Windows Telemetry ETW Monitor). 
+
+The output of the TIV framework is a report in the form of a web page.
+
+## Technology domain
+
+The TIV framework is based on Python 3.5.
+
+## Usage guidelines
+
+In order to run the TIV framework, execute the "visualize_data.py" Python script. This script is the central engine of TIV.
+
+The "visualize_data.py" script takes the following parameters: 
+
+- *-i*: specifies the input file. This file is the file to which the Windbg Framework has logged output data;
+- *-o*: specifies the output directory. In this directory (which must not exist), TIV stores output files; 
+- *--debug* (optional): enables verbose output.
+
+Once the "visualize_data.py" script has finished executing, a new folder is created in the folder specified by the *-o* parameter. The name of the newly created folder is the folder creation timestamp. This folder stores the report displaying information and statistics in the form of a web page (default page: "index.html").
+
+In order to view the report, the content of the newly created folder has to be served by an HTTP server. The simplest way to achieve this is to use the native Python HTTP server. Execute the following command in the newly created folder: 
+
+(Python 2.7)
+
+```bash
+python -m SimpleHTTPServer <port>
+```
+
+(Python 3.5)
+
+```bash
+python -m http.server <port>
+```
+
+**Important note**: Opening the "index.html" file directly will fail due to CORS.
+
+### Log output
+
+Log output produced by the TIV framework can be found in the "tiv\log.txt" file.
+
+## Extending the TIV framework
+
+### Adding new statistics
+
+In order to create a new statistic, you must follow these steps:
+
+1. In the "tiv\statistics" folder, create your own object as a subclass of the "Statistic" class 
+2. Implement the two functions that are mandatory for statistic objects: *build_statistic* and *get_rendered_format* (take a look at the implementation of the "Statistic" class)
+3. Add an import line in the "statisticHelper.py" file (stored in the "tiv\statistics" folder)
+
+### Adding new graphs
+
+In order to create a new graph, you must follow these steps:
+
+1. In the "tiv\graphs" folder, create your own object as a subclass of the "Graph" class
+2. Implement the two functions that are mandatory for graph objects: *build_graph_data* and *get_graph_data* (take a look at the implementation of the "Graph" class)
+3. Add an import line in the "graphsHelper.py" file (stored in the "tiv\graphs" folder)
+
+## Credits
+
+**Pablo Artuso**
+
+⋅⋅⋅ Main contributor
+
+⋅⋅⋅ Email: artusopablo@gmail.com
+
+⋅⋅⋅ Twitter account: [@lmkalg](https://twitter.com/lmkalg)
+
+**Aleksandar Milenkoski** 
+
+⋅⋅⋅ Supervisor and contributor
+
+⋅⋅⋅ Email: amilenkoski@ernw.de
+
+⋅⋅⋅ Twitter account: [@milenkowski](https://twitter.com/milenkowski)
+

files/wintel_etwmonitor/tiv/core/callStack.py

@@ -0,0 +1,25 @@
+from misc.constants import CALL_SITE, CALL_STACK_ERROR, CALL_STACK_NO_INFO
+class CallStack:
+    def __init__(self):
+        self.call_stack = []
+
+    def parse(self, callstack_data):
+        for call in callstack_data.split("\n"):
+            if (call.strip()
+               and not CALL_SITE in call.lower().strip()
+               and not CALL_STACK_ERROR in call.lower().strip() 
+               and not CALL_STACK_NO_INFO in call.lower().strip()):
+                self.call_stack.append(call.split(" ")[1])
+
+    def get_call_stack(self):
+        return self.call_stack
+
+    def get_call_stack_as_string(self):
+        return " <- ".join(self.call_stack)
+
+    def __str__(self):
+        output = ""
+        for call in self.call_stack:
+            output += "{}\n".format(call)
+        return output
+

files/wintel_etwmonitor/tiv/core/date.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+from misc.constants import DATE_FORMAT
+class Date:
+    def __init__(self):
+        self.date = None
+
+    def get_date(self):
+        return self.date
+
+    def parse(self, date_data):
+        """
+            Debugger (not debuggee) time: Mon Jun  4 11:29:31.215 2018 (UTC + 2:00)
+        """     
+        # Get the value of the line
+        date_line = date_data
+        # Remove prefix of date
+        date_line = ":".join((date_line.split(":")[1:])).rstrip()
+        # Remove the timezone part
+        date_line = date_line[:date_line.index('(')].strip()
+        self.date = datetime.strptime(date_line, DATE_FORMAT)
+
+    def __str__(self):
+        return self.date.strftime(DATE_FORMAT)

files/wintel_etwmonitor/tiv/core/eventDescriptor.py

@@ -0,0 +1,79 @@
+from misc.constants import  ED_ID,ED_VERSION,ED_CHANNEL,ED_LEVEL,ED_OPCODE,ED_TASK,ED_KEYWORD
+class EventDescriptor:
+    def __init__(self):
+        self.id = None
+        self.version = None
+        self.channel = None
+        self.level = None
+        self.opcode = None
+        self.task = None
+        self.keyword = None
+
+    def get_id(self):
+        return self.id
+
+    def get_version(self):
+        return self.version
+
+    def get_channel(self):
+        return self.channel
+
+    def get_level(self):
+        return self.level
+
+    def get_opcode(self):
+        return self.opcode
+
+    def get_task(self):
+        return self.task
+
+    def get_keyword(self):
+        return self.keyword
+
+    def _parse_event_descriptor_value(self, value):
+        if "'" in value:
+            return int(value.split(" ")[0],16)
+        elif "`" in value:
+            return int(value.replace("`","").strip(),16)
+        else:
+            return int(value.strip(),16)
+
+    def parse(self, event_descriptor_data):
+        """
+            +0x000 Id               : 0x30ea
+            +0x002 Version          : 0 ''
+            +0x003 Channel          : 0xb ''
+            +0x004 Level            : 0x5 ''
+            +0x005 Opcode           : 0 ''
+            +0x006 Task             : 0
+            +0x008 Keyword          : 0x00008000`00000000
+        """
+
+        for line in event_descriptor_data.split('\n'):
+            if line:    
+                address_and_attribute, value = line.split(":")
+                address, attribute = address_and_attribute.strip().split(" ")
+                real_value = self._parse_event_descriptor_value(value.strip())
+                if ED_ID in attribute.lower():
+                    self.id = real_value
+                elif ED_VERSION in attribute.lower():
+                    self.version = real_value
+                elif ED_CHANNEL in attribute.lower():
+                    self.channel = real_value
+                elif ED_LEVEL in attribute.lower():
+                    self.level = real_value
+                elif ED_OPCODE in attribute.lower():
+                    self.opcode = real_value
+                elif ED_TASK in attribute.lower():
+                    self.task = real_value
+                elif ED_KEYWORD in attribute.lower():
+                    self.keyword = real_value
+                else:
+                    raise Exception("Invalid attribute when trying to parse Event Descriptor data. Line: {}".format(line))
+
+
+    def __str__(self):
+        output = ""
+        for key, value in self.__dict__.items():
+            output += "{}: {}\n".format(key,hex(value))
+        return output

files/wintel_etwmonitor/tiv/core/eventJsonFormat.py

@@ -0,0 +1,63 @@
+from core.exceptions import ErrorWhenParsing
+from misc.constants import (OPENING_BRACKET, EJF_FOUND_BUFFERS, CLOSING_BRACKET, OPENING_BRACE,
+                      CLOSING_BRACE, EJF_NO_BUFFERS_FOUND)
+class EventJsonFormat:
+    def __init__(self):
+        self.event_json = None
+        self.sender = None
+
+    def get_event_json(self):
+        return self.event_json
+
+    def get_sender(self):
+        return self.sender
+
+    def _parse_line_with_event_json_format(self, line_with_event_json_format):
+        # Skip first the characters
+        line_with_event_json_format = line_with_event_json_format[3:]
+        index_of_opening_bracket = line_with_event_json_format.index(OPENING_BRACKET)
+        index_of_closing_bracket = line_with_event_json_format.index(CLOSING_BRACKET)
+        index_of_opening_brace = line_with_event_json_format.index(OPENING_BRACE)
+        index_of_closing_brace = line_with_event_json_format.index(CLOSING_BRACE)
+
+        self.sender = line_with_event_json_format[index_of_opening_bracket+1:index_of_closing_bracket]
+        self.event_json = line_with_event_json_format[index_of_opening_brace:index_of_closing_brace+1]
+
+
+    def parse(self,event_json_format_data):
+        """
+            (WmiTrace) LogDump for Logger Id 0x21
+            Reading Buffer   1 /   4
+            Reading Buffer   2 /   4
+            Reading Buffer   3 /   4
+            Reading Buffer   4 /   4
+            Found Buffers: 4 Messages: 2, sorting entries
+            [0]0AB4.0AF4::  131804496926295220 [Microsoft-Windows-Shell-NotificationCenter/ActionCenterButtonStateOnLaunching/]{"messageState": 0, "notificationState": 1}
+            Total of 2 Messages from 4 Buffers
+        """
+        line_with_event_json_format = ""
+        index = 0
+        index_of_found_buffers = -1
+        for line in event_json_format_data.split("\n"):
+            if EJF_FOUND_BUFFERS in line.lower():
+                index_of_found_buffers = index 
+                break
+            elif EJF_NO_BUFFERS_FOUND in line.lower():
+                self.sender = "<NO_DATA_FOUND>"
+                self.event_json = "<NO_DATA_FOUND>"
+                return 
+            else:
+                index += 1
+        if index_of_found_buffers == -1:
+            raise ErrorWhenParsing("\" Found buffers\" wasn't found in the Json event format provided: {}".format(event_json_format_data))
+        line_with_event_json_format = event_json_format_data.split("\n")[index_of_found_buffers+1]
+
+        self._parse_line_with_event_json_format(line_with_event_json_format)
+
+    def __str__(self):
+        output = ""
+        for key, value in self.__dict__.items():
+            output += "{}: {}\n".format(key,value)
+        return output
+
+

files/wintel_etwmonitor/tiv/core/exceptions.py

@@ -0,0 +1,2 @@
+class ErrorWhenParsing(Exception):
+    pass

files/wintel_etwmonitor/tiv/core/graphData.py

@@ -0,0 +1,50 @@
+class GraphData:
+    def __init__(self, name, _type, values, labels, label, title, filename, display_legend, with_time):
+        self.name = name
+        self._type = _type
+        self.values = values
+        self.labels = labels
+        self.label = label
+        self.title  = title 
+        self.filename = filename
+        self.id = name
+        self.display_legend = display_legend
+        self.with_time = with_time
+
+    def get_name(self):
+        return self.name
+
+    def get_type(self):
+        return self._type
+
+    def get_values(self):
+        return self.values
+
+    def get_labels(self):
+        return self.labels
+
+    def get_label(self):
+        return self.label
+
+    def get_title(self):
+        return self.title
+
+    def get_filename(self):
+        return self.filename
+
+    def get_id(self):
+        return self.id
+
+    def get_display_legend(self):
+        return self.display_legend
+
+    def get_with_time(self):
+        return self.with_time
+
+
+
+
+
+
+
+