work in progress for idaholab#329, improve extracted_files download page

mmguero-dev · Feb 5, 2024 · 27185b4 · 27185b4
1 parent f2e47f1
commit 27185b4
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 13 deletions.
diff --git a/file-monitor/scripts/extracted_files_http_server.py b/file-monitor/scripts/extracted_files_http_server.py
@@ -15,7 +15,7 @@
 import re
 import sys
 from Crypto.Cipher import AES
-from datetime import datetime, UTC
+from datetime import datetime, timedelta, UTC
 from dominate.tags import *
 from http.server import HTTPServer, SimpleHTTPRequestHandler
 from socketserver import ThreadingMixIn
@@ -81,14 +81,16 @@ def do_GET(self):
         fullpath, relpath = self.translate_path(self.path)
         fileBaseName = os.path.basename(fullpath)
 
-        nowStr = datetime.now(UTC).isoformat()
+        tomorrowStr = (datetime.now(UTC) + timedelta(days=1)).isoformat().split('.')[0]
 
         # HTTP-FUID-UID-TIMESTAMP.ext
         carvedFileRegex = re.compile(
             r'^(?P<source>[^-]+)-(?P<fuid>F[a-zA-Z0-9]+|unknown)-(?P<uid>C[a-zA-Z0-9]+|unknown)-(?P<timestamp>\d+)(?P<ext>\..+)?$'
         )
         # UID-FUID-whatever
         carvedFileRegexAlt = re.compile(r'^(?P<uid>C[a-zA-Z0-9]+)_(?P<fuid>F[a-zA-Z0-9]+)')
+        # XOR decrypted from FEieEe1f1SI6YJk4H5
+        xorRegex = re.compile(r'^(?P<source>XOR) decrypted from (?P<fuid>F[a-zA-Z0-9]+)')
 
         if os.path.isdir(fullpath) and (args.links or (not os.path.islink(fullpath))):
             # directory listing
@@ -110,6 +112,7 @@ def do_GET(self):
 
             # <body>
             with doc:
+                # header decoration
                 with nav(cls='navbar navbar-light bg-light static-top'):
                     div(cls='container')
                 header(cls='masthead')
@@ -171,6 +174,7 @@ def do_GET(self):
                                                 # calculate some of the stuff for representing Malcolm files
                                                 timestamp = None
                                                 timestampStr = ''
+                                                timestampStartFilterStr = ''
                                                 fmatch = None
                                                 fsource = ''
                                                 fids = list()
@@ -185,19 +189,33 @@ def do_GET(self):
                                                         try:
                                                             timestamp = datetime.strptime(timestampStr, '%Y%m%d%H%M%S')
                                                             timestampStr = timestamp.isoformat()
+                                                            timestampStartFilterStr = (
+                                                                (timestamp - timedelta(days=1))
+                                                                .isoformat()
+                                                                .split('.')[0]
+                                                            )
                                                         except Exception as te:
                                                             if timestampStr:
                                                                 eprint(f'Error with time "{str(timestampStr)}": {te}')
-                                                        fsource = fmatch.groupdict().get('source', '')
+                                                        # put UIDs and FUIDs into a single event.id-filterable column
                                                         fids = list(
-                                                            filter(
-                                                                None,
-                                                                [
+                                                            [
+                                                                x
+                                                                for x in [
                                                                     fmatch.groupdict().get('uid', ''),
                                                                     fmatch.groupdict().get('fuid', ''),
-                                                                ],
-                                                            )
+                                                                ]
+                                                                if x and x != 'unknown'
+                                                            ]
                                                         )
+                                                        # massage source a little bit (remove '<error>' and handle
+                                                        #   'XOR decrypted from...')
+                                                        fsource = fmatch.groupdict().get('source', '')
+                                                        if fsource == '<error>':
+                                                            fsource = ''
+                                                        elif xorMatch := xorRegex.search(fsource):
+                                                            fsource = xorMatch.groupdict().get('source', '')
+                                                            fids.append(xorMatch.groupdict().get('fuid', ''))
 
                                                 # only request mime type for files if specified in arguments
                                                 fileinfo = (
@@ -237,14 +255,14 @@ def do_GET(self):
                                                     # list carve source, IDs, and timestamp
                                                     t.add(
                                                         td(
-                                                            fmatch.groupdict().get('source', ''),
+                                                            fsource,
                                                             style="text-align: center",
                                                         ),
                                                         td(
                                                             [
                                                                 a(
                                                                     fid,
-                                                                    href=f'/arkime/idark2dash/filter?start={timestampStr}&stop={nowStr}&field=event.id&value={fid}',
+                                                                    href=f'/arkime/idark2dash/filter?start={timestampStartFilterStr}&stop={tomorrowStr}&field=event.id&value={fid}',
                                                                 )
                                                                 for fid in fids
                                                             ],
@@ -270,6 +288,7 @@ def do_GET(self):
                                     # our "walk" is not recursive right now, we only need to go one level deep
                                     break
 
+                # footer decoration
                 with footer(cls='footer bg-light').add(div(cls='container')).add(div(cls='row')):
                     with div(cls="col-lg-6 h-100 text-center text-lg-start my-auto"):
                         p(

diff --git a/nginx/landingpage/index.html b/nginx/landingpage/index.html
@@ -22,7 +22,9 @@
             <div class="container"/>
         </nav>
         <!-- Masthead-->
-        <header class="masthead"/>
+        <header class="masthead">
+            <div class="container"/>
+        </header>
         <!-- Icons Grid-->
         <section class="features-icons bg-light text-center">
             <div class="container">

diff --git a/nginx/nginx_idark2dash_rewrite_dashboards.conf b/nginx/nginx_idark2dash_rewrite_dashboards.conf
@@ -1,4 +1,4 @@
-rewrite ^.*/idark2dash/(.*) $dashboards_prefix/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'$sessions_index',interval:auto,query:(language:lucene,query:''),sort:!($time_field,desc)) redirect;
+rewrite ^.*/idark2dash/(.*) $dashboards_prefix/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,to:$filter_stop_time))&_a=(description:'',filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))),fullScreenMode:!f,options:(useMargins:!t),query:(language:lucene,query:'*'),timeRestore:!f,viewMode:view) redirect;
 proxy_pass $dashboards_proxy_pass;
 proxy_redirect off;
 proxy_set_header Host dashboards.malcolm.local;
diff --git a/nginx/nginx_idark2dash_rewrite_kibana.conf b/nginx/nginx_idark2dash_rewrite_kibana.conf
@@ -1 +1 @@
-rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'$sessions_index',interval:auto,query:(language:lucene,query:''),sort:!($time_field,desc)) redirect;
+rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(refreshInterval:(pause:!t,value:60000),time:(from:'2024-02-01T15:45:45.793Z',to:'2024-02-06T16:00:50.775Z'))&_a=(filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))))? redirect;