chg: [tracker] add experimental report generator

bin/lib/Tracker.py

@@ -1055,6 +1055,23 @@ def api_delete_tracker(data, user_id):
     tracker = Tracker(tracker_uuid)
     return tracker.delete(), 200
 
+def api_tracker_add_object(data, user_id):
+    tracker_uuid = data.get('uuid')
+    res = api_check_tracker_acl(tracker_uuid, user_id)
+    if res:
+        return res
+    tracker = Tracker(tracker_uuid)
+    object_gid = data.get('gid')
+    date = data.get('date')
+    if date:
+        if not Date.validate_str_date(date):
+            date = None
+    try:
+        obj_type, subtype, obj_id = object_gid.split(':', 2)
+    except (AttributeError, IndexError):
+        return {"status": "error", "reason": "Invalid Object"}, 400
+    return tracker.add(obj_type, subtype, obj_id, date=date), 200
+
 def api_tracker_remove_object(data, user_id):
     tracker_uuid = data.get('uuid')
     res = api_check_tracker_acl(tracker_uuid, user_id)

bin/lib/chats_viewer.py

@@ -322,7 +322,7 @@ def get_threads_metas(threads):
 def get_username_meta_from_global_id(username_global_id):
     _, instance_uuid, username_id = username_global_id.split(':', 2)
     username = Usernames.Username(username_id, instance_uuid)
-    return username.get_meta()
+    return username.get_meta(options={'icon'})
 
 # TODO Filter
 ## Instance type
@@ -386,6 +386,8 @@ def get_user_account_chats_meta(user_id, chats, subchannels):
         c_subtype, c_id = chat_g_id.split(':', 1)
         chat = Chats.Chat(c_id, c_subtype)
         chat_meta = chat.get_meta(options={'icon', 'info', 'nb_participants', 'tags_safe', 'username'})
+        if chat_meta['username']:
+            chat_meta['username'] = get_username_meta_from_global_id(chat_meta['username'])
         chat_meta['nb_messages'] = len(chat.get_user_messages(user_id))
         chat_meta['subchannels'] = []
         for subchannel_gid in chat.get_subchannels():
@@ -425,6 +427,39 @@ def get_user_account_nb_all_week_messages(user_id, chats, subchannels):
         nb_day += 1
     return stats
 
+def _get_chat_card_meta_options():
+    return {'created_at', 'icon', 'info', 'nb_participants', 'origin_link', 'subchannels', 'tags_safe', 'threads', 'translation', 'username'}
+
+def _get_message_bloc_meta_options():
+    return {'chat', 'content', 'files-names', 'icon', 'images', 'language', 'link', 'parent', 'parent_meta', 'reactions','thread', 'translation', 'user-account'}
+
+def get_message_report(l_mess): # TODO Force language + translation
+    translation_target = 'en'
+    chats = {}
+    messages = []
+    mess_options = _get_message_bloc_meta_options()
+
+    l_mess = sorted(l_mess, key=lambda x: x[2])
+
+    for m in l_mess:
+        message = Messages.Message(m[2])
+        meta = message.get_meta(options=mess_options, translation_target=translation_target)
+        if meta['chat'] not in chats:
+            chat = Chats.Chat(meta['chat'], message.get_chat_instance())
+            meta_chat = chat.get_meta(options=_get_chat_card_meta_options(), translation_target=translation_target)
+            if meta_chat['username']:
+                meta_chat['username'] = get_username_meta_from_global_id(meta_chat['username'])
+            chats[chat.id] = meta_chat
+
+            # stats
+            chats[chat.id]['t_messages'] = 1
+        else:
+            chats[meta['chat']]['t_messages'] += 1
+
+        messages.append(meta)
+
+    return chats, messages
+
 #### FIX ####
 
 def fix_correlations_subchannel_message():

bin/lib/objects/Chats.py

@@ -56,6 +56,13 @@ def get_link(self, flask_context=False):
             url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}'
         return url
 
+    def get_origin_link(self):
+        if self.subtype == '00098785-7e70-5d12-a120-c5cdc1252b2b':
+            username = self.get_username()
+            if username:
+                username = username.split(':', 2)[2]
+                return f'https://t.me/{username}'
+
     def get_svg_icon(self):  # TODO
         # if self.subtype == 'telegram':
         #     style = 'fab'
@@ -100,6 +107,8 @@ def get_meta(self, options=set(), translation_target=None):
             meta['threads'] = self.get_threads()
         if 'tags_safe' in options:
             meta['tags_safe'] = self.is_tags_safe(meta['tags'])
+        if 'origin_link' in options:
+            meta['origin_link'] = self.get_origin_link()
         return meta
 
     def get_misp_object(self):

bin/lib/objects/Messages.py

@@ -71,6 +71,10 @@ def get_source(self):
     def get_basename(self):
         return os.path.basename(self.id)
 
+    def get_chat_instance(self):
+        c_id = self.id.split('/')
+        return c_id[0]
+
     def get_content(self, r_type='str'): # TODO ADD cache # TODO Compress content ???????
         """
         Returns content
@@ -259,7 +263,7 @@ def get_meta(self, options=None, timestamp=None, translation_target=''):
         else:
             timestamp = float(timestamp)
         timestamp = datetime.utcfromtimestamp(float(timestamp))
-        meta['date'] = timestamp.strftime('%Y/%m/%d')
+        meta['date'] = timestamp.strftime('%Y-%m-%d')
         meta['hour'] = timestamp.strftime('%H:%M:%S')
         meta['full_date'] = timestamp.isoformat(' ')
         if 'last_full_date' in options:

bin/lib/objects/UsersAccount.py

@@ -150,7 +150,7 @@ def get_meta(self, options=set(), translation_target=None): # TODO Username time
             if meta['username']:
                 _, username_account_subtype, username_account_id = meta['username'].split(':', 3)
                 if 'username_meta' in options:
-                    meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta()
+                    meta['username'] = Usernames.Username(username_account_id, username_account_subtype).get_meta(options={'icon'})
                 else:
                     meta['username'] = {'type': 'username', 'subtype': username_account_subtype, 'id': username_account_id}
         if 'usernames' in options:

bin/packages/Date.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import datetime
+import time
 from calendar import monthrange
 
 from dateutil.rrule import rrule, MONTHLY
@@ -91,6 +92,10 @@ def get_current_week_day():
     start = dt - datetime.timedelta(days=dt.weekday())
     return start.strftime("%Y%m%d")
 
+def get_current_utc_full_time():
+    timestamp = datetime.datetime.fromtimestamp(time.time())
+    return timestamp.strftime('%Y-%m-%d %H:%M:%S')
+
 def get_month_dates(date=None):
     if date:
         date = convert_date_str_to_datetime(date)

var/www/blueprints/correlation.py

@@ -359,6 +359,10 @@ def show_relationship():
                 dict_object["metadata"]['type_id'] = subtype
             else:
                 dict_object["subtype"] = ''
-                dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id)
+            dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id)
+            dict_object["metadata_card"]['tags_safe'] = True
             return render_template("show_relationship.html", dict_object=dict_object, bootstrap_label=bootstrap_label,
-                                       tags_selector_data=Tag.get_tags_selector_data())
+                                   tags_selector_data=Tag.get_tags_selector_data(),
+                                   meta=dict_object["metadata_card"],
+                                   ail_tags=dict_object["metadata_card"]["add_tags_modal"])
+

var/www/blueprints/hunters.py

@@ -24,6 +24,7 @@
 ##################################
 from lib import ail_core
 from lib.objects import ail_objects
+from lib import chats_viewer
 from lib import item_basic
 from lib import Tracker
 from lib import Tag
@@ -372,6 +373,27 @@ def get_json_tracker_graph():
         res = Tracker.get_trackers_graph_by_day([tracker_uuid])
     return jsonify(res)
 
+@hunters.route('/tracker/object/add', methods=['GET'])
+@login_required
+@login_admin
+def tracker_object_add():
+    user_id = current_user.get_id()
+    tracker_uuid = request.args.get('uuid')
+    object_global_id = request.args.get('gid')
+    if object_global_id.startswith('messages::'):
+        obj = ail_objects.get_obj_from_global_id(object_global_id)
+        date = obj.get_date()
+    else:
+        date = request.args.get('date')  # TODO check daterange
+    res = Tracker.api_tracker_add_object({'uuid': tracker_uuid, 'gid': object_global_id, 'date': date}, user_id)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
+
 @hunters.route('/tracker/object/remove', methods=['GET'])
 @login_required
 @login_analyst
@@ -389,6 +411,41 @@ def tracker_object_remove():
             return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
 
 
+@hunters.route('/tracker/objects', methods=['GET'])
+@login_required
+@login_admin
+def tracker_objects():
+    user_id = current_user.get_id()
+    tracker_uuid = request.args.get('uuid', None)
+    res = Tracker.api_is_allowed_to_edit_tracker(tracker_uuid, user_id)
+    if res[1] != 200:  # invalid access
+        return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+    tracker = Tracker.Tracker(tracker_uuid)
+    meta = tracker.get_meta(options={'description', 'sparkline', 'tags', 'nb_objs'})
+    if meta['type'] == 'yara':
+        yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
+    else:
+        yara_rule_content = None
+
+    chats, messages = chats_viewer.get_message_report(tracker.get_objs())
+
+    meta['date'] = Date.get_current_utc_full_time()
+
+    return render_template("messages_report.html", meta=meta, yara_rule_content=yara_rule_content,
+                           chats=chats, messages=messages, bootstrap_label=bootstrap_label)
+
+    # TODO
+
+    # Manual - Title
+    #        - Summary
+
+    # Messages table
+
+    # Timeline messages by chats - line
+    # pie charts NB messages all chats
+    # Barchart NB messages by days
+
 ####################
 #    RETRO HUNT    #
 ####################

var/www/templates/chats_explorer/basic_card_chat.html

@@ -13,58 +13,87 @@ <h4 class="text-secondary mb-0">
                     <text x="15" y="15" text-anchor="middle" dominant-baseline="central" class="{{ meta["svg_icon"]["style"] }}" font-size="16px">{{ meta["svg_icon"]["icon"] }}</text>
                 </g>
             </svg>
-            {% if meta['username'] %}{{ meta["username"]["id"] }} {% else %} {{ meta['name'] }}{% endif %} :  <small><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></small>
+            {% if meta['name'] %}{{ meta['name'] }}{% endif %}{% if not report_mode %} :  <small><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?subtype={{ meta['subtype'] }}&id={{ meta['id'] }}">{{ meta['id'] }}</a></small>{% endif %}
         </h4>
     </div>
     <div class="card-body py-0">
-        <span class="">
-            {% if meta["tags_safe"] %}
-                {% if meta['icon'] %}
-                    <span><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="my-1" alt="{{ meta['id'] }}" width="200" height="200"></span>
+
+        <div class="d-flex align-items-center">
+            <div>
+                {% if meta["tags_safe"] %}
+                    {% if meta['icon'] %}
+                        <span><img src="{{ url_for('objects_image.image', filename=meta['icon'])}}" class="my-1" alt="{{ meta['id'] }}" width="200" height="200"></span>
+                    {% endif %}
+                {% else %}
+                    <span class="my-2 fa-stack fa-8x">
+                        <i class="fas fa-stack-1x fa-image"></i>
+                        <i class="fas fa-stack-2x fa-ban" style="color:Red"></i>
+                    </span>
                 {% endif %}
-            {% else %}
-                <span class="my-2 fa-stack fa-8x">
-                    <i class="fas fa-stack-1x fa-image"></i>
-                    <i class="fas fa-stack-2x fa-ban" style="color:Red"></i>
-                </span>
-            {% endif %}
-        </span>
+            </div>
 
-        <span>
-            <span class="badge badge-dark">
-                <span class="badge badge-info" style="font-size: 0.8rem;">
-                    <i class="fas fa-hourglass-start"></i>
-                </span>
-                {{meta["first_seen"]}}
-                <span class="badge badge-light mx-1" style="font-size: 1rem;">
-                    <i class="far fa-calendar-alt"></i>
-                </span>
-                {{meta["last_seen"]}}
-                <span class="badge badge-secondary" style="font-size: 0.8rem;">
-                    <i class="fas fa-hourglass-end"></i>
-                </span>
-            </span>
-            <span class="badge badge-dark">
-                <span class="badge badge-info" style="font-size: 0.8rem;">
-                    <i class="far fa-comments"></i>
-                </span>
-                {{meta["nb_subchannels"]}}&nbsp;&nbsp;
-                <span class="badge badge-info" style="font-size: 0.8rem;">
-                    <i class="fas fa-user-circle"></i>
-                </span>
-                {{meta["nb_participants"]}}
-            </span>
-            <span class="badge badge-dark">
-                <span class="badge badge-info" style="font-size: 0.8rem;">
-                    <i class="fas fa-user-circle"></i>
-                    <i class="far fa-comment-dots"></i>
-                </span>
-                {{meta["nb_messages"]}}&nbsp;&nbsp;
-            </span>
-        </span>
 
-        <div class="">
-            {{ meta['info'] }}
+            <div>
+                {% if meta['username'] %}
+                    <div class="mx-2">
+                        <svg height="30" width="30">
+                            <g class="nodes">
+                                <circle cx="15" cy="15" r="15" fill="{{ meta["username"]["icon"]["color"] }}"></circle>
+                                <text x="15" y="15" text-anchor="middle" dominant-baseline="central" class="{{ meta["username"]["icon"]["style"] }}" font-size="16px">{{ meta["username"]["icon"]["icon"] }}</text>
+                            </g>
+                        </svg>
+                        {{ meta['username']['id'] }}
+                    </div>
+                {% endif %}
+
+
+                <div class="badge badge-dark mx-2 my-1">
+                    <span class="badge badge-info" style="font-size: 0.8rem;">
+                        <i class="fas fa-hourglass-start"></i>
+                    </span>
+                    {{meta["first_seen"][0:4]}}-{{meta["first_seen"][4:6]}}-{{meta["first_seen"][6:8]}}
+                    <span class="badge badge-light mx-1" style="font-size: 1rem;">
+                        <i class="far fa-calendar-alt"></i>
+                    </span>
+                    {{meta["last_seen"][0:4]}}-{{meta["last_seen"][4:6]}}-{{meta["last_seen"][6:8]}}
+                    <span class="badge badge-secondary" style="font-size: 0.8rem;">
+                        <i class="fas fa-hourglass-end"></i>
+                    </span>
+                </div>
+{#                <div class="mx-2">#}
+{#                    <span class="badge badge-dark">#}
+{#                        <span class="badge badge-info" style="font-size: 0.8rem;">#}
+{#                            <i class="fas fa-calendar-plus"></i>#}
+{#                        </span>#}
+{#                        {{meta["created_at"]}}#}
+{#                    </span>#}
+{#                </div>#}
+                <div class="mx-2">
+                    <span class="badge badge-dark">
+                        <span class="badge badge-info" style="font-size: 0.8rem;">
+                            <i class="far fa-comments"></i> Subchannels
+                        </span>
+                        {{meta["nb_subchannels"]}}&nbsp;&nbsp;
+                        <span class="badge badge-info" style="font-size: 0.8rem;">
+                            <i class="fas fa-user-circle"></i> Participants
+                        </span>
+                        {{meta["nb_participants"]}}
+                    </span>
+                    {% if "nb_messages" in meta %}
+                        <span class="badge badge-dark">
+                            <span class="badge badge-info" style="font-size: 0.8rem;">
+                                <i class="fas fa-user-circle"></i>
+                                <i class="far fa-comment-dots"></i>
+                            </span>
+                            {{ meta["nb_messages"] }}&nbsp;&nbsp;
+                        </span>
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+
+        <div>
+            <pre class="my-0" style="white-space: pre-wrap;">{{ meta['info'] }}</pre>
         </div>
 
         <div class="">
@@ -120,5 +149,7 @@ <h4 class="text-secondary mb-0">
         {% endif %}
 
 	</div>
-    {% include 'objects/block_object_footer_small.html' %}
+    {% if not report_mode %}
+        {% include 'objects/block_object_footer_small.html' %}
+    {% endif %}
 </div>