Merge branch 'privacy'

doc/source/api.rst

@@ -472,6 +472,24 @@ Thug API interface definition is reported below for convenience.
             @return: None
             """
 
+        def get_web_tracking():
+            """
+            get_web_tracking
+
+            Return True if web client tracking inspection is enabled, False otherwise.
+
+            @return: bool
+            """
+
+        def set_web_tracking():
+            """
+            set_web_tracking
+
+            Enable web client tracking inspection
+
+            @return: None
+            """
+
         def add_urlclassifier(rule):
             """
             add_urlclassifier

doc/source/usage.rst

@@ -46,6 +46,7 @@ Let's start our Thug tour by taking a look at the options it provides.
         -B, --broken-url        Set the broken URL mode
         -y, --vtquery           Query VirusTotal for samples analysis
         -s, --vtsubmit          Submit samples to VirusTotal
+        -z, --web-tracking      Enable web client tracking inspection
         -N, --no-honeyagent     Disable HoneyAgent support
 
         Plugins:

src/DOM/HTTPSession.py

@@ -135,8 +135,9 @@ def build_http_headers(self, window, personality, headers):
         if window and window.url not in ('about:blank', ):
             http_headers['Referer'] = self.normalize_url(window, window.url)
 
-        if window and window.doc.cookie:
-            http_headers['Cookie'] = window.doc.cookie
+        # REVIEW ME!
+        #if window and window.doc.cookie:
+        #    http_headers['Cookie'] = window.doc.cookie
 
         for name, value in headers.items():
             http_headers[name] = value
@@ -165,6 +166,7 @@ def fetch(self, url, method = "GET", window = None, personality = None, headers
                            verify  = False)
 
         self.filecount += 1
+        log.WebTracking.inspect_response(response)
         return response
 
     def threshold_expired(self, url):

src/DOM/WebTracking.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+#
+# WebTracking.py
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA  02111-1307  USA
+
+import datetime
+import logging
+
+log = logging.getLogger("Thug")
+
+
+MAX_COOKIE_EXPIRES_DAYS = 365 
+
+
+class WebTracking(object):
+    cookie_expires_delta = datetime.timedelta(days = MAX_COOKIE_EXPIRES_DAYS)
+
+    def __init__(self):
+        self.now = datetime.datetime.now()
+
+    def _do_inspect_cookies(self, response):
+        for cookie in response.cookies:
+            expires = datetime.datetime.fromtimestamp(cookie.expires)
+            if self.now + self.cookie_expires_delta < expires:
+                log.ThugLogging.log_warning("[PRIVACY] Cookie expiring at %s (more than %s days from now)" % (expires, 
+                                                                                                              MAX_COOKIE_EXPIRES_DAYS, ))
+
+    def _inspect_cookies(self, response):
+        if response.history:
+            for r in response.history:
+                self._do_inspect_cookies(r)
+
+        self._do_inspect_cookies(response)
+
+    def inspect_response(self, response):
+        if not log.ThugOpts.web_tracking:
+            return
+
+        self._inspect_cookies(response)

src/ThugAPI/IThugAPI.py

@@ -475,6 +475,24 @@ def set_vt_submit():
         @return: None
         """
 
+    def get_web_tracking():
+        """
+        get_web_tracking
+
+        Return True if web client tracking inspection is enabled, False otherwise.
+
+        @return: bool
+        """
+
+    def set_web_tracking():
+        """
+        set_web_tracking
+
+        Enable web client tracking inspection
+
+        @return: None
+        """
+
     def add_urlclassifier(rule):
         """
         add_urlclassifier

src/ThugAPI/ThugAPI.py

@@ -27,7 +27,12 @@
     import urlparse
 
 from DOM.W3C import w3c
-from DOM import Window, HTTPSession, DFT, MIMEHandler, SchemeHandler
+from DOM import Window
+from DOM import HTTPSession
+from DOM import WebTracking
+from DOM import DFT
+from DOM import MIMEHandler
+from DOM import SchemeHandler
 from Encoding import Encoding
 from Logging.ThugLogging import ThugLogging
 
@@ -54,6 +59,7 @@ def __init__(self, args):
         self.thug_version       = __thug_version__
         log.ThugOpts            = ThugOpts()
         log.ThugVulnModules     = ThugVulnModules()
+        log.WebTracking         = WebTracking.WebTracking()
         log.MIMEHandler         = MIMEHandler.MIMEHandler()
         log.SchemeHandler       = SchemeHandler.SchemeHandler()
         log.JSClassifier        = JSClassifier.JSClassifier()
@@ -179,6 +185,12 @@ def get_broken_url(self):
     def set_broken_url(self):
         log.ThugOpts.broken_url = True
 
+    def get_web_tracking(self):
+        return log.ThugOpts.web_tracking
+
+    def set_web_tracking(self):
+        log.ThugOpts.web_tracking = True
+
     def disable_honeyagent(self):
         log.ThugOpts.honeyagent = False
 

src/ThugAPI/ThugOpts.py

@@ -55,6 +55,7 @@ def __init__(self):
         self._broken_url      = False
         self._vt_query        = False
         self._vt_submit       = False
+        self._web_tracking    = False
         self._honeyagent      = True
         self._cache           = '/tmp/thug-cache-%s' % (os.getuid(), )
         self.Personality      = Personality()
@@ -212,6 +213,14 @@ def set_vt_submit(self):
 
     vt_submit = property(get_vt_submit, set_vt_submit)
 
+    def get_web_tracking(self):
+        return self._web_tracking
+
+    def set_web_tracking(self, enabled):
+        self._web_tracking = enabled
+
+    web_tracking = property(get_web_tracking, set_web_tracking)
+
     def get_honeyagent(self):
         return self._honeyagent
 

src/thug.py

@@ -64,6 +64,7 @@ def usage(self):
         -B, --broken-url    \tSet the broken URL mode
         -y, --vtquery       \tQuery VirusTotal for samples analysis
         -s, --vtsubmit      \tSubmit samples to VirusTotal
+        -z, --web-tracking  \tEnable web client tracking inspection
         -N, --no-honeyagent \tDisable HoneyAgent support
 
         Plugins:
@@ -100,7 +101,7 @@ def analyze(self):
 
         try:
             options, args = getopt.getopt(self.args,
-                                          'hVu:e:w:n:o:r:p:ysNlxvdqmagA:PS:RJ:Kt:ET:BQ:W:C:FZM',
+                                          'hVu:e:w:n:o:r:p:yszNlxvdqmagA:PS:RJ:Kt:ET:BQ:W:C:FZM',
                 ['help',
                 'version',
                 'useragent=',
@@ -112,6 +113,7 @@ def analyze(self):
                 'proxy=',
                 'vtquery',
                 'vtsubmit',
+                'web-tracking',
                 'no-honeyagent',
                 'local',
                 'local-nofetch',
@@ -165,6 +167,8 @@ def analyze(self):
                 self.set_vt_query()
             if option[0] in ('-s', '--vtsubmit', ):
                 self.set_vt_submit()
+            if option[0] in ('-z', '--web-tracking', ):
+                self.set_web_tracking()
             if option[0] in ('-N', '--no-honeyagent', ):
                 self.disable_honeyagent()
             if option[0] in ('-l', '--local', ):