Restyled and extended Web UI, fixed some bugs

appmon.py

@@ -21,6 +21,7 @@
 from termcolor import colored
 import database as db
 import platform as platform_module
+from urllib.parse import unquote
 
 print("""
      ___      .______   .______   .___  ___.   ______   .__   __.
@@ -35,6 +36,7 @@
 
 app = Flask(__name__, static_url_path='/static')
 #app.debug = True
+console_logging = True
 
 device = ''
 session = ''
@@ -56,30 +58,12 @@ def add_header(r):
     return r
 
 
-@app.route('/api/fetch', methods=['GET'])
-def serve_json():
-    index = request.args.get('id')
-    if request.args.get('reportdb'):
-        db_name = request.args.get('reportdb')
-    else:
-        db_name = request.args.get('app')
-    response = db.read_from_database(db_name, index)
-    #response = open('static/data.json').read()
-    return response
-
-
-@app.route('/monitor/', methods=['GET'])
-def monitor_page():
-    app_name = request.args.get('app')
-    return render_template('monitor.html', app_name=app_name)
-
-
 @app.route('/', methods=['GET'])
 def landing_page():
     global APP_LIST, DB_MAP
+    APP_LIST.clear()
 
-    app_dumps_dir = os.path.join('.','app_dumps')
-    for root, dirs, files in os.walk(app_dumps_dir):
+    for root, dirs, files in os.walk(output_dir):
         path = root.split(os.sep)
         for file in files:
             file_path = os.path.join(root, file)
@@ -89,6 +73,40 @@ def landing_page():
     return render_template('index.html', apps=APP_LIST)
 
 
+@app.route('/monitor/', methods=['GET'])
+def monitor_page():
+    app_name = request.args.get('app')
+    return render_template('monitor.html', app_name=app_name)
+
+
+@app.route('/api/fetch', methods=['GET'])
+def read_db():
+    index = request.args.get('id')
+    if request.args.get('reportdb'):
+        db_name = request.args.get('reportdb')
+    else:
+        db_name = request.args.get('app')
+    grouped = request.args.get('grouped')
+    db_name = unquote(db_name)
+    db_path = os.path.join(output_dir, str(db_name) + '.db')
+    #print('db_path: %s, index: %s' % (db_path, index), flush=True)
+    response = db.read_from_database(db_path, index, grouped)
+    #response = open('static/data.json').read()
+    return response
+
+@app.route('/api/clear_table', methods=['GET'])
+def clear_db():
+    if request.args.get('reportdb'):
+        db_name = request.args.get('reportdb')
+    else:
+        db_name = request.args.get('app')
+    # URL decode
+    db_name = unquote(db_name)
+    db_path = os.path.join(output_dir, str(db_name) + '.db')
+    response = str(db.delete_all_from_table(db_path))
+    return response
+
+
 def init_opts():
     parser = argparse.ArgumentParser()
     parser.add_argument('-a', action='store', dest='app_name', default='',
@@ -196,9 +214,11 @@ def on_message(message, data):
         db.save_to_database(writePath, message['payload'])
         #writePath = os.path.join(output_dir, app_name + '.json')
         #writeBinFile(writePath, message['payload']) #writeBinFile(writePath, binascii.unhexlify(message['payload']))
-        print((colored('[%s] Dumped to %s' % (current_time, writePath), 'green')))
+        if console_logging:
+            print(colored('[%s] Dumped to %s' % (current_time, writePath), 'green'), flush=True)
+            print(message['payload'], flush=True)
     elif message['type'] == 'error':
-        print((message['stack']))
+        print(colored("[INTERNAL FRIDA ERROR]: \n" + message['stack'], "red"), flush=True)
 
 
 def generate_injection():

database/__init__.py

@@ -18,26 +18,38 @@
 import dataset, json, time, htmlentities
 import platform as platform_module
 from xml.sax.saxutils import escape
+from termcolor import colored
+
+table_name = 'api_captures'
 
 def save_to_database(db_path, str_json):
   try:
     str_json = json.loads(str_json.replace("\n", "<br />").replace("\r", "<br />"), strict=False)
     db = dataset.connect('sqlite:///%s' % (db_path.replace("'", "_")))
-    table = db['api_captures']
-    os_string = platform_module.system()
-    if os_string == "Windows":
-        formatted_time = time.strftime('%b %d %Y %I:%M %p', time.localtime())
-    else:
-        formatted_time = time.strftime('%b %d %Y %l:%M %p', time.localtime())
-    table.insert(dict(time=formatted_time,
+    table = db[table_name]
+    table.insert(dict(time=time.strftime('%b %d %Y %H:%M:%S', time.localtime()),
       operation=str_json['txnType'],
       artifact=json.dumps(str_json['artifact']),
       method=str_json['method'],
       module=str_json['lib'],
       remark=''))
   except Exception as e:
-    print(str(e))
-    print(str_json)
+    print(str(e), flush=True)
+    print(str_json, flush=True)
+
+def delete_all_from_table(db_path):
+  try:
+    db = dataset.connect('sqlite:///%s' % (db_path.replace("'", "_") + '?check_same_thread=False'))
+    table = db[table_name]
+    db.begin()
+    table.delete()
+    db.commit()
+    db.close()
+    print(colored('Table "%s" in database "%s" has been cleared!' % (table_name, db_path), 'cyan'), flush=True)
+    return True
+  except Exception as e:
+    print(str(e), flush=True)
+    return False
 
 def stringify(data):
   str_data = ""
@@ -50,19 +62,22 @@ def stringify(data):
     except Exception as e:
       return data
 
-def read_from_database(db_path, index=0):
+def read_from_database(db_path, index=0, grouped_by=''):
   result_set = {}
   parent_holder = []
-  db = dataset.connect('sqlite:///./app_dumps/%s.db' % (db_path))
-  api_captures = db.query('SELECT * FROM api_captures GROUP BY artifact')
+  db = dataset.connect('sqlite:///%s' % (db_path.replace("'", "_") + '?check_same_thread=False'))
+  query_str = 'SELECT * FROM ' + table_name + (' GROUP BY ' + grouped_by if grouped_by != '' else '')
+  api_captures = db.query(query_str)
   for capture in api_captures:
-    child_holder = []
-    child_holder.append(capture['operation'])
-    child_holder.append(capture['module'])
-    child_holder.append(capture['method'])
+    child_holder = {}
+    child_holder['id'] = capture['id']
+    child_holder['time'] = capture['time']
+    child_holder['operation'] = capture['operation']
+    child_holder['module'] = capture['module']
+    child_holder['method'] = capture['method']
     str_artifact = ''
     artifacts = json.loads(capture['artifact'])
-    
+
     for artifact in artifacts:
       if "name" in artifact:
         artifact_name = artifact['name']
@@ -81,10 +96,8 @@ def read_from_database(db_path, index=0):
 
     #print str_artifact
 
-    child_holder.append(str_artifact)
-    child_holder.append(capture['time'])
-    child_holder.append(capture['id'])
-    child_holder.append(capture['remark'])
+    child_holder['artifact'] = str_artifact
+    child_holder['remark'] = capture['remark']
     parent_holder.append(child_holder)
   result_set['data'] = parent_holder
   return json.dumps(result_set)