Csv files #1

deploy/docker-compose.yaml

@@ -6,14 +6,14 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
     ports:
-      - 22181:2181
+      - 22182:2181
 
   kafka:
     image: confluentinc/cp-kafka:latest
     depends_on:
       - zookeeper
     ports:
-      - 29092:29092
+      - 29093:29092
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

deploy/requirements.txt

@@ -1 +1,2 @@
-kafka-python==2.0.2
+kafka-python==2.0.2
+pandas==2.1.4

src/log_analyser/log_analyser.py

@@ -0,0 +1,97 @@
+import os
+from datetime import datetime, timedelta
+from objects.match import Match
+
+
+class LogAnalyser:
+
+    def __init__(self, path_csv, name) -> None:
+
+        self.path_csv = path_csv
+        self.name = name
+
+        self.date = self.name2datetime()
+
+        self.match = None
+
+        self.actions = {"match_start": self.process_match_start}
+
+    def run(self):
+
+        with open(self.path_csv, encoding='utf-8') as my_file:
+            file = my_file.read()
+            lines = file.split("\n")
+            for line in lines:
+                line_split = line.split(",")
+
+                if len(line_split) > 1:
+                    type = line_split[1]
+                    if type in self.actions:
+                        self.actions[type](line_split)
+
+        with open("../logs_process/{}.json".format(self.name.split(".")[0]), "w") as file:
+            file.write(self.match.export_json())
+
+
+    def name2datetime(self):
+
+        date_string = self.name.split(".")[0].split("Log-")[1]
+        date_object = datetime.strptime(date_string, '%Y-%m-%d-%H-%M-%S')
+
+        return date_object
+
+    def process_match_start(self, data):
+
+        self.match = Match.from_json({"rounds": [], 
+                       "date": self.date,
+                       "map_name": data[3],
+                       "map_type": data[4],
+                       "team1_name": data[5],
+                       "team2_name": data[6],
+                       "score_team1": 0,
+                       "score_team2": 0,
+                       })
+
+        self.actions = {"match_start": self.process_match_start,
+                        "round_start": self.match.add_round,
+                        "round_end": self.match.end_round,
+                        "hero_spawn": self.process_hero_spawn,
+                        "hero_swap": self.process_hero_swap,
+                        "kill": self.match.add_kill,
+                        "ultimate_charged": self.match.add_ultimate_charged,
+                        "ultimate_start": self.match.add_ultimate_start,
+                        "ultimate_end": self.match.add_ultimate_end,
+                        "objective_captured": self.match.add_objective_captured,
+                        "player_stat": self.match.add_player_stat,
+                        "point_progress": self.match.add_objective_progress,
+                        "payload_progress": self.match.add_objective_progress,
+                        }
+
+    def process_hero_spawn(self, data):
+
+        player_data = {"time": data[2], "team_name": data[3], "player_name": data[4], "character_name": data[5]}
+        self.match.add_player(player_data)
+
+    def process_hero_swap(self, data):
+
+        hero_data = {"time": data[2], "team_name": data[3], "player_name": data[4], "character_name": data[5], "character_swap": data[6]}
+        self.match.add_hero_swap(hero_data)
+
+    def convert_timefile_to_datetime(self, time_string):
+
+        # Utilisation de strptime pour convertir la chaîne en datetime
+        time_delta = datetime.strptime(time_string, "[%H:%M:%S]")
+
+        # Conversion en timedelta (représentation de la durée)
+        duration = timedelta(hours=time_delta.hour, minutes=time_delta.minute, seconds=time_delta.second)
+        return duration
+
+
+for file in os.listdir("../logs"):
+    if file.endswith(".txt"):
+        print(file)
+        la = LogAnalyser('../logs/{}'.format(file), file)
+        la.run()
+
+# la = LogAnalyser('../logs/Log-2023-12-22-21-12-32.txt', "Log-2023-12-22-21-12-32.txt")
+# la.run()

src/log_analyser/objects/character.py

@@ -0,0 +1,48 @@
+from objects.object import Object
+
+
+class Character(Object):
+    def __init__(self, **kwargs):
+
+        data_schema = {"name": str,
+                       "stats": dict,
+                       "played_time": list,
+                       "kills": list,
+                       "deads": list,
+                       "offensive_assists": list,
+                       "defensive_assists": list,
+                       "ultimate_charged": list,
+                       "ultimate_use": list}
+
+        super().__init__(data_schema, **kwargs)
+
+
+    def add_played_time(self, data):
+        self.played_time.append(data)
+
+    def add_kill(self, data):
+        self.kills.append(data)
+
+    def add_death(self, data):
+        self.deads.append(data)
+
+    def add_offensive_assist(self, data):
+        self.offensive_assists.append(data)
+
+    def add_defensive_assist(self, data):
+        self.defensive_assists.append(data)
+
+    def add_ultimate_charged(self, data):
+        self.ultimate_charged.append(data)
+
+    def add_ultimate_start(self, data):
+        self.ultimate_use.append(data)
+
+    def add_ultimate_end(self, data):
+
+        if len(self.ultimate_use) == 0:
+            return -1
+        self.ultimate_use[-1]["end"] = data["end"]
+
+    def add_character_stats(self, data):
+        self.stats = data

src/log_analyser/objects/match.py

@@ -0,0 +1,160 @@
+from objects.object import Object
+from objects.round import Round
+from objects.team import Team
+from datetime import datetime
+
+
+class Match(Object):
+
+    def __init__(self, **kwargs):
+
+        data_schema = {"rounds": list,
+                       "date": datetime,
+                       "map_name": str,
+                       "map_type": str,
+                       "team1_name": str,
+                       "team2_name": str,
+                       "score_team1": int,
+                       "score_team2": int,
+                       }
+
+        super().__init__(data_schema, **kwargs)
+
+        self.actual_round = -1
+
+    def add_round(self, data):
+
+        teams = {}
+        teams[self.team1_name] = Team.from_json({"name": self.team1_name, "players": {}})
+        teams[self.team2_name] = Team.from_json({"name": self.team2_name, "players": {}})
+        self.rounds.append(Round.from_json({"teams": teams, "start_time": data[2], "objective_captured": [], "objective_progress": []}))
+
+        self.actual_round += 1
+
+        print("###### NEW ROUND {} #######".format(self.actual_round))
+
+    def add_player(self, data):
+
+        if data["player_name"] in self.rounds[self.actual_round].teams[data["team_name"]].players:
+            self.add_character(data)
+            return -1
+        else:
+            self.rounds[self.actual_round].teams[data["team_name"]].add_player(
+                {"name": data["player_name"], "characters": {}})
+            # print("add player", data["player_name"])
+            self.add_character(data)
+            return 0
+
+    def add_character(self, data):
+
+        if data["character_name"] in self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters:
+            return -2
+        else:
+            self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].add_character({"name": data["character_name"], "stats": {}, "played_time": [], "kills": [], "deads": [], "offensive_assists": [], "defensive_assists": [], "ultimate_charged": [], "ultimate_use": []})
+
+        if len(self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time) > 0:
+            self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time[-1]["end"] = data["time"]
+
+        self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].add_played_time({"start": data["time"]})
+
+    def add_kill(self, data):
+
+        killer_data = {"time": data[2], "player_victim": data[7], "character_victim": data[8]}
+        victim_data = {"time": data[2], "player_killer": data[4], "character_killer": data[5]}
+
+        self.create_if_player_and_caracter_not_exist(data[3], data[4], data[5])
+        self.create_if_player_and_caracter_not_exist(data[6], data[7], data[8])
+
+        if not data[5] in self.rounds[self.actual_round].teams[data[3]].players[data[4]].characters:
+            self.add_character({"time": data[2], "team_name": data[3], "player_name": data[4], "character_name": data[5]})
+
+        if not data[6] in self.rounds[self.actual_round].teams[data[6]].players[data[7]].characters:
+            self.add_character({"time": data[2], "team_name": data[6], "player_name": data[7], "character_name": data[8]})
+
+        self.rounds[self.actual_round].teams[data[3]].players[data[4]].characters[data[5]].add_kill(killer_data)
+        self.rounds[self.actual_round].teams[data[6]].players[data[7]].characters[data[8]].add_death(victim_data)
+
+    def add_player_stat(self, data):
+
+        # self.create_if_player_and_caracter_not_exist(data[4], data[5], data[6])
+
+        player_data = {"eliminations": data[6], "final_blows": data[7], "deaths": data[8], "damage": data[9],
+                       "barrier_damage": data[10], "hero_damage": data[11], "healing": data[12], "healing_receive": data[13],
+                       "self_healing": data[14], "damage_taken": data[15], "damage_blocked": data[16], "defensive_assist": data[17],
+                       "offensive_assists": data[18], "ultimated_earn": data[19], "ultimates_used": data[20], "solo_kills": data[23],
+                       "critical_hits_accuracy": data[28], "weapon_accuracy": data[37], "hero_time_played": data[38]}
+
+        if player_data["hero_time_played"] != "0" and data[6] in self.rounds[self.actual_round].teams[data[4]].players[data[5]].characters:
+            self.rounds[self.actual_round].teams[data[4]].players[data[5]].characters[data[6]].add_character_stats(player_data)
+
+    def add_hero_swap(self, data):
+
+        self.add_player(data)
+        if data["character_swap"] in self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters:
+            self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_swap"]].add_played_time({"end": data["time"]})
+
+
+    def create_if_player_and_caracter_not_exist(self, team, player_name, character_name):
+
+        if not player_name in self.rounds[self.actual_round].teams[team].players:
+            self.add_player({"team_name": team, "player_name": player_name, "character_name": character_name, "time": self.rounds[self.actual_round].start_time})
+
+        if not character_name in self.rounds[self.actual_round].teams[team].players[player_name].characters:
+            self.add_character({"team_name": team, "player_name": player_name, "character_name": character_name, "time": self.rounds[self.actual_round].start_time})
+
+    def add_ultimate_start(self, data):
+
+        self.create_if_player_and_caracter_not_exist(data[3], data[4], data[5])
+
+        ultimate_start_data = {"start": data[2]}
+        self.rounds[self.actual_round].teams[data[3]].players[data[4]].characters[data[5]].add_ultimate_start(ultimate_start_data)
+
+    def add_ultimate_end(self, data):
+
+        self.create_if_player_and_caracter_not_exist(data[3], data[4], data[5])
+
+        ultimate_end_data = {"end": data[2]}
+        self.rounds[self.actual_round].teams[data[3]].players[data[4]].characters[data[5]].add_ultimate_end(ultimate_end_data)
+
+    def add_ultimate_charged(self, data):
+
+        self.create_if_player_and_caracter_not_exist(data[3], data[4], data[5])
+
+        ultimate_charged_data = data[2]
+        self.rounds[self.actual_round].teams[data[3]].players[data[4]].characters[data[5]].add_ultimate_charged(ultimate_charged_data)
+
+    def add_objective_captured(self, data):
+
+        objective_capture_data = {"time": data[2], "team_name": data[4], "control_team1_progress": data[6],
+                                  "control_team2_progress": data[7]}
+
+        self.rounds[self.actual_round].add_objective_captured(objective_capture_data)
+
+
+    def add_objective_progress(self, data):
+
+        objectif_progress_data = {"time": data[2], "team_name": data[4], "progress": data[6]}
+        self.rounds[self.actual_round].add_objective_progress(objectif_progress_data)
+        pass
+
+    def end_round(self, data):
+
+        end_round_data = {"time": data[2], "score_team1": data[5], "score_team2": data[6]}
+
+        for team in self.rounds[self.actual_round].teams:
+            for player in self.rounds[self.actual_round].teams[team].players:
+                for character in self.rounds[self.actual_round].teams[team].players[player].characters:
+                    if not "end" in self.rounds[self.actual_round].teams[team].players[player].characters[character].played_time[-1]:
+                        self.rounds[self.actual_round].teams[team].players[player].characters[character].played_time[-1]["end"] = end_round_data["time"]
+
+
+        self.score_team1 = end_round_data["score_team1"]
+        self.score_team2 = end_round_data["score_team2"]
+        print("###### END ROUND {} #######\n".format(self.actual_round))
+
+    def end_match(self, data):
+
+        end_match_data = {"time": data[2], "score_team1": data[4], "score_team2": data[5]}
+
+        self.score_team1 = end_match_data["score_team1"]
+        self.score_team2 = end_match_data["score_team2"]

src/log_analyser/objects/object.py

@@ -0,0 +1,66 @@
+import json
+from datetime import datetime, timedelta
+
+class Object:
+    def __init__(self, class_model, **kwargs):   
+
+        self.from_json_ok = True
+        self.class_name = self.__class__.__name__
+
+        for key, value in kwargs.items():
+            if key in class_model:
+                if type(value) == class_model[key]:
+                    setattr(self, key, value)
+                else:
+                    print("type of key {} is invalid".format(class_model, key))
+                    self.from_json_ok = False
+            else:
+                print("key {} not find in dataschema".format(key))
+                self.from_json_ok = False
+
+        for key, value in class_model.items():
+            if not hasattr(self, key):
+                print("key {} not find in object".format(key, value()))
+                setattr(self, key, value())
+
+    def export_json_recursive(self, data):
+        if issubclass(type(data), Object):
+            dict_class = data.__dict__.copy()
+            dict_class.pop("from_json_ok")
+            return data.export_json_recursive(dict_class)
+        elif isinstance(data, datetime):
+            return data.strftime("%Y-%m-%d %H:%M:%S")
+        elif isinstance(data, list):
+            return [self.export_json_recursive(item) for item in data]
+        elif isinstance(data, dict):
+            return {key: self.export_json_recursive(value) for key, value in data.items()}
+        else:
+            return data
+
+    def export_json(self):
+
+        dict_class = self.__dict__.copy()
+        dict_class.pop("from_json_ok")
+        for key, value in dict_class.items():
+            dict_class[key] = self.export_json_recursive(value)
+
+        return json.dumps(dict_class, indent=4, sort_keys=True, default=str)
+
+    def convert_timefile_to_datetime(self, time_string):
+
+
+        # Utilisation de strptime pour convertir la chaîne en datetime
+        time_delta = datetime.strptime(time_string, "[%H:%M:%S]")
+
+        # Conversion en timedelta (représentation de la durée)
+        duration = timedelta(hours=time_delta.hour, minutes=time_delta.minute, seconds=time_delta.second)
+        return duration
+
+    @classmethod
+    def from_json(self, data):
+        object = self(**data)
+        if object.from_json_ok:
+            return object
+        else:
+            return None
+