diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index 737a42e..477a838 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -13,7 +13,7 @@ services: depends_on: - zookeeper ports: - - 29093:29092 + - 29092:29092 environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 diff --git a/src/kafka_lib.py b/src/kafka_lib.py index a449a75..e2fc014 100644 --- a/src/kafka_lib.py +++ b/src/kafka_lib.py @@ -18,8 +18,6 @@ def __init__(self, url, date=None, nb_try_connection=10): self.date = date self.nb_try_connection = nb_try_connection - - self.init_producer_ok, self.producer = self.init_producer(url) self.queue_sender = queue.Queue(2) diff --git a/src/log_analyser/log_analyser.py b/src/log_analyser/log_analyser.py index 512c011..548d037 100644 --- a/src/log_analyser/log_analyser.py +++ b/src/log_analyser/log_analyser.py @@ -1,6 +1,6 @@ import os from datetime import datetime, timedelta -from objects.match import Match +from log_analyser.objects.map import Map class LogAnalyser: @@ -12,9 +12,9 @@ def __init__(self, path_csv, name) -> None: self.date = self.name2datetime() - self.match = None + self.map = None - self.actions = {"match_start": self.process_match_start} + self.actions = {"match_start": self.process_map_start} def run(self): @@ -29,8 +29,8 @@ def run(self): if type in self.actions: self.actions[type](line_split) - with open("../logs_process/{}.json".format(self.name.split(".")[0]), "w") as file: - file.write(self.match.export_json()) + # with open("../logs_process/{}.json".format(self.name.split(".")[0]), "w") as file: + # file.write(self.map.export_json()) def name2datetime(self): @@ -40,9 +40,9 @@ def name2datetime(self): return date_object - def process_match_start(self, data): + def process_map_start(self, data): - self.match = Match.from_json({"rounds": [], + self.map = Map.from_json({"rounds": [], "date": self.date, "map_name": data[3], "map_type": data[4], @@ -52,30 +52,30 @@ def process_match_start(self, data): "score_team2": 0, }) - self.actions = {"match_start": self.process_match_start, - "round_start": self.match.add_round, - "round_end": self.match.end_round, + self.actions = {"match_start": self.process_map_start, + "round_start": self.map.add_round, + "round_end": self.map.end_round, "hero_spawn": self.process_hero_spawn, "hero_swap": self.process_hero_swap, - "kill": self.match.add_kill, - "ultimate_charged": self.match.add_ultimate_charged, - "ultimate_start": self.match.add_ultimate_start, - "ultimate_end": self.match.add_ultimate_end, - "objective_captured": self.match.add_objective_captured, - "player_stat": self.match.add_player_stat, - "point_progress": self.match.add_objective_progress, - "payload_progress": self.match.add_objective_progress, + "kill": self.map.add_kill, + "ultimate_charged": self.map.add_ultimate_charged, + "ultimate_start": self.map.add_ultimate_start, + "ultimate_end": self.map.add_ultimate_end, + "objective_captured": self.map.add_objective_captured, + "player_stat": self.map.add_player_stat, + "point_progress": self.map.add_objective_progress, + "payload_progress": self.map.add_objective_progress, } def process_hero_spawn(self, data): player_data = {"time": data[2], "team_name": data[3], "player_name": data[4], "character_name": data[5]} - self.match.add_player(player_data) + self.map.add_player(player_data) def process_hero_swap(self, data): hero_data = {"time": data[2], "team_name": data[3], "player_name": data[4], "character_name": data[5], "character_swap": data[6]} - self.match.add_hero_swap(hero_data) + self.map.add_hero_swap(hero_data) def convert_timefile_to_datetime(self, time_string): @@ -87,11 +87,11 @@ def convert_timefile_to_datetime(self, time_string): return duration -for file in os.listdir("../logs"): - if file.endswith(".txt"): - print(file) - la = LogAnalyser('../logs/{}'.format(file), file) - la.run() +# for file in os.listdir("../logs"): +# if file.endswith(".txt"): +# print(file) +# la = LogAnalyser('../logs/{}'.format(file), file) +# la.run() # la = LogAnalyser('../logs/Log-2023-12-22-21-12-32.txt', "Log-2023-12-22-21-12-32.txt") # la.run() \ No newline at end of file diff --git a/src/log_analyser/objects/character.py b/src/log_analyser/objects/character.py index d82538c..d8742ea 100644 --- a/src/log_analyser/objects/character.py +++ b/src/log_analyser/objects/character.py @@ -1,4 +1,4 @@ -from objects.object import Object +from log_analyser.objects.object import Object class Character(Object): @@ -8,9 +8,7 @@ def __init__(self, **kwargs): "stats": dict, "played_time": list, "kills": list, - "deads": list, - "offensive_assists": list, - "defensive_assists": list, + "deaths": list, "ultimate_charged": list, "ultimate_use": list} @@ -24,7 +22,7 @@ def add_kill(self, data): self.kills.append(data) def add_death(self, data): - self.deads.append(data) + self.deaths.append(data) def add_offensive_assist(self, data): self.offensive_assists.append(data) diff --git a/src/log_analyser/objects/match.py b/src/log_analyser/objects/map.py similarity index 94% rename from src/log_analyser/objects/match.py rename to src/log_analyser/objects/map.py index 0a3e14e..c0ac064 100644 --- a/src/log_analyser/objects/match.py +++ b/src/log_analyser/objects/map.py @@ -1,10 +1,10 @@ -from objects.object import Object -from objects.round import Round -from objects.team import Team +from log_analyser.objects.object import Object +from log_analyser.objects.round import Round +from log_analyser.objects.team import Team from datetime import datetime -class Match(Object): +class Map(Object): def __init__(self, **kwargs): @@ -50,7 +50,7 @@ def add_character(self, data): if data["character_name"] in self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters: return -2 else: - self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].add_character({"name": data["character_name"], "stats": {}, "played_time": [], "kills": [], "deads": [], "offensive_assists": [], "defensive_assists": [], "ultimate_charged": [], "ultimate_use": []}) + self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].add_character({"name": data["character_name"], "stats": {}, "played_time": [], "kills": [], "deaths": [], "ultimate_charged": [], "ultimate_use": []}) if len(self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time) > 0: self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time[-1]["end"] = data["time"] @@ -152,9 +152,9 @@ def end_round(self, data): self.score_team2 = end_round_data["score_team2"] print("###### END ROUND {} #######\n".format(self.actual_round)) - def end_match(self, data): + def end_map(self, data): - end_match_data = {"time": data[2], "score_team1": data[4], "score_team2": data[5]} + end_map_data = {"time": data[2], "score_team1": data[4], "score_team2": data[5]} - self.score_team1 = end_match_data["score_team1"] - self.score_team2 = end_match_data["score_team2"] + self.score_team1 = end_map_data["score_team1"] + self.score_team2 = end_map_data["score_team2"] diff --git a/src/log_analyser/objects/object.py b/src/log_analyser/objects/object.py index f782372..1b3e6ca 100644 --- a/src/log_analyser/objects/object.py +++ b/src/log_analyser/objects/object.py @@ -6,6 +6,7 @@ def __init__(self, class_model, **kwargs): self.from_json_ok = True self.class_name = self.__class__.__name__ + self.class_model = class_model for key, value in kwargs.items(): if key in class_model: @@ -25,9 +26,15 @@ def __init__(self, class_model, **kwargs): def export_json_recursive(self, data): if issubclass(type(data), Object): + dict_class = data.__dict__.copy() - dict_class.pop("from_json_ok") - return data.export_json_recursive(dict_class) + dict_class_final = dict_class.copy() + + for key, value in dict_class.items(): + if not key in data.class_model and key != "class_name": + dict_class_final.pop(key) + + return data.export_json_recursive(dict_class_final) elif isinstance(data, datetime): return data.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(data, list): @@ -40,12 +47,16 @@ def export_json_recursive(self, data): def export_json(self): dict_class = self.__dict__.copy() - dict_class.pop("from_json_ok") - for key, value in dict_class.items(): - dict_class[key] = self.export_json_recursive(value) - return json.dumps(dict_class, indent=4, sort_keys=True, default=str) + dict_class_final = dict_class.copy() + for key, value in dict_class.items(): + if not key in self.class_model and key != "class_name": + dict_class_final.pop(key) + else: + dict_class_final[key] = self.export_json_recursive(value) + # return json.dumps(dict_class_final, indent=4, sort_keys=True, default=str) + return json.dumps(dict_class_final, skipkeys=False, ensure_ascii=True, check_circular=True) def convert_timefile_to_datetime(self, time_string): diff --git a/src/log_analyser/objects/player.py b/src/log_analyser/objects/player.py index c6266fa..c8277a1 100644 --- a/src/log_analyser/objects/player.py +++ b/src/log_analyser/objects/player.py @@ -1,6 +1,6 @@ # from src.log_analyser.character import Character -from objects.object import Object -from objects.character import Character +from log_analyser.objects.object import Object +from log_analyser.objects.character import Character class Player(Object): diff --git a/src/log_analyser/objects/round.py b/src/log_analyser/objects/round.py index f08062a..4ab375e 100644 --- a/src/log_analyser/objects/round.py +++ b/src/log_analyser/objects/round.py @@ -1,5 +1,5 @@ -from objects.object import Object -from objects.team import Team +from log_analyser.objects.object import Object +from log_analyser.objects.team import Team class Round(Object): diff --git a/src/log_analyser/objects/team.py b/src/log_analyser/objects/team.py index e0f022c..20db965 100644 --- a/src/log_analyser/objects/team.py +++ b/src/log_analyser/objects/team.py @@ -1,5 +1,6 @@ -from objects.object import Object -from objects.player import Player +from log_analyser.objects.object import Object +from log_analyser.objects.player import Player + class Team(Object): diff --git a/src/main.py b/src/main.py index 20d5d0c..9d96bca 100644 --- a/src/main.py +++ b/src/main.py @@ -2,18 +2,18 @@ import time import sys from kafka_lib import ProducerThread, ConsumerThread +from log_analyser.log_analyser import LogAnalyser class DatastrikePythonProcessing: def __init__(self): self.running = True + + self.producer_thread = ProducerThread("localhost:29092") - - self.producer_thread = ProducerThread("localhost:29093") - - self.consumer_thread = ConsumerThread("localhost:29093") - self.consumer_thread.add_topics("test", self.on_callback_test) + self.consumer_thread = ConsumerThread("localhost:29092") + self.consumer_thread.add_topics("analyse", self.on_callback_test) self.consumer_thread.start() self.producer_thread.start() @@ -21,12 +21,15 @@ def __init__(self): def on_callback_test(self, topic, data): print("message receive : ", topic, data) - print("a") + + la = LogAnalyser('logs/Log-2023-12-22-21-12-32.txt', "Log-2023-12-22-21-12-32.txt") + la.run() + self.producer_thread.send("analyse.report", la.map.export_json()) def run(self): while self.running: - print("Service en cours d'exécution...") + # print("Service en cours d'exécution...") time.sleep(1) print("Service stop.")