From 6fb667a2d6a2363600516200dd021ad2c302873d Mon Sep 17 00:00:00 2001 From: btheytaz Date: Fri, 5 Jan 2024 17:15:20 +0100 Subject: [PATCH 1/5] export_json export only key present in data_schema of each object now --- src/log_analyser/objects/character.py | 2 -- src/log_analyser/objects/match.py | 2 +- src/log_analyser/objects/object.py | 23 +++++++++++++++++------ src/main.py | 3 +-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/log_analyser/objects/character.py b/src/log_analyser/objects/character.py index d82538c..0d12e78 100644 --- a/src/log_analyser/objects/character.py +++ b/src/log_analyser/objects/character.py @@ -9,8 +9,6 @@ def __init__(self, **kwargs): "played_time": list, "kills": list, "deads": list, - "offensive_assists": list, - "defensive_assists": list, "ultimate_charged": list, "ultimate_use": list} diff --git a/src/log_analyser/objects/match.py b/src/log_analyser/objects/match.py index 0a3e14e..2e74d39 100644 --- a/src/log_analyser/objects/match.py +++ b/src/log_analyser/objects/match.py @@ -50,7 +50,7 @@ def add_character(self, data): if data["character_name"] in self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters: return -2 else: - self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].add_character({"name": data["character_name"], "stats": {}, "played_time": [], "kills": [], "deads": [], "offensive_assists": [], "defensive_assists": [], "ultimate_charged": [], "ultimate_use": []}) + self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].add_character({"name": data["character_name"], "stats": {}, "played_time": [], "kills": [], "deads": [], "ultimate_charged": [], "ultimate_use": []}) if len(self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time) > 0: self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time[-1]["end"] = data["time"] diff --git a/src/log_analyser/objects/object.py b/src/log_analyser/objects/object.py index f782372..987f975 100644 --- a/src/log_analyser/objects/object.py +++ b/src/log_analyser/objects/object.py @@ -6,6 +6,7 @@ def __init__(self, class_model, **kwargs): self.from_json_ok = True self.class_name = self.__class__.__name__ + self.class_model = class_model for key, value in kwargs.items(): if key in class_model: @@ -25,9 +26,15 @@ def __init__(self, class_model, **kwargs): def export_json_recursive(self, data): if issubclass(type(data), Object): + dict_class = data.__dict__.copy() - dict_class.pop("from_json_ok") - return data.export_json_recursive(dict_class) + dict_class_final = dict_class.copy() + + for key, value in dict_class.items(): + if not key in data.class_model: + dict_class_final.pop(key) + + return data.export_json_recursive(dict_class_final) elif isinstance(data, datetime): return data.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(data, list): @@ -40,11 +47,15 @@ def export_json_recursive(self, data): def export_json(self): dict_class = self.__dict__.copy() - dict_class.pop("from_json_ok") - for key, value in dict_class.items(): - dict_class[key] = self.export_json_recursive(value) - return json.dumps(dict_class, indent=4, sort_keys=True, default=str) + dict_class_final = dict_class.copy() + + for key, value in dict_class.items(): + if not key in self.class_model: + dict_class_final.pop(key) + else: + dict_class_final[key] = self.export_json_recursive(value) + return json.dumps(dict_class_final, indent=4, sort_keys=True, default=str) def convert_timefile_to_datetime(self, time_string): diff --git a/src/main.py b/src/main.py index 20d5d0c..6bc02a1 100644 --- a/src/main.py +++ b/src/main.py @@ -8,8 +8,7 @@ class DatastrikePythonProcessing: def __init__(self): self.running = True - - + self.producer_thread = ProducerThread("localhost:29093") self.consumer_thread = ConsumerThread("localhost:29093") From 37af9fb1e5201d5b179e6223159e3d3b11ee0426 Mon Sep 17 00:00:00 2001 From: btheytaz Date: Fri, 5 Jan 2024 17:16:04 +0100 Subject: [PATCH 2/5] change deads to deaths --- src/log_analyser/objects/character.py | 2 +- src/log_analyser/objects/match.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log_analyser/objects/character.py b/src/log_analyser/objects/character.py index 0d12e78..62e251a 100644 --- a/src/log_analyser/objects/character.py +++ b/src/log_analyser/objects/character.py @@ -8,7 +8,7 @@ def __init__(self, **kwargs): "stats": dict, "played_time": list, "kills": list, - "deads": list, + "deaths": list, "ultimate_charged": list, "ultimate_use": list} diff --git a/src/log_analyser/objects/match.py b/src/log_analyser/objects/match.py index 2e74d39..cda2f79 100644 --- a/src/log_analyser/objects/match.py +++ b/src/log_analyser/objects/match.py @@ -50,7 +50,7 @@ def add_character(self, data): if data["character_name"] in self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters: return -2 else: - self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].add_character({"name": data["character_name"], "stats": {}, "played_time": [], "kills": [], "deads": [], "ultimate_charged": [], "ultimate_use": []}) + self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].add_character({"name": data["character_name"], "stats": {}, "played_time": [], "kills": [], "deaths": [], "ultimate_charged": [], "ultimate_use": []}) if len(self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time) > 0: self.rounds[self.actual_round].teams[data["team_name"]].players[data["player_name"]].characters[data["character_name"]].played_time[-1]["end"] = data["time"] From 92d38057a91d774f972fa3185e891a832bdda785 Mon Sep 17 00:00:00 2001 From: benjamin Date: Sat, 6 Jan 2024 17:28:41 +0100 Subject: [PATCH 3/5] send message to backend --- deploy/docker-compose.yaml | 2 +- src/main.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index 737a42e..477a838 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -13,7 +13,7 @@ services: depends_on: - zookeeper ports: - - 29093:29092 + - 29092:29092 environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 diff --git a/src/main.py b/src/main.py index 6bc02a1..6f66a25 100644 --- a/src/main.py +++ b/src/main.py @@ -9,10 +9,10 @@ def __init__(self): self.running = True - self.producer_thread = ProducerThread("localhost:29093") + self.producer_thread = ProducerThread("localhost:29092") - self.consumer_thread = ConsumerThread("localhost:29093") - self.consumer_thread.add_topics("test", self.on_callback_test) + self.consumer_thread = ConsumerThread("localhost:29092") + self.consumer_thread.add_topics("analyse", self.on_callback_test) self.consumer_thread.start() self.producer_thread.start() @@ -20,7 +20,8 @@ def __init__(self): def on_callback_test(self, topic, data): print("message receive : ", topic, data) - print("a") + + self.producer_thread.send("analyse_report", "test") def run(self): From 3a6b0a7113890e3b73947e1e34f3e03e0c9393b1 Mon Sep 17 00:00:00 2001 From: benjamin Date: Sun, 7 Jan 2024 19:45:04 +0100 Subject: [PATCH 4/5] associate main service and log analyser (launch analysis with kafka) --- src/kafka_lib.py | 2 -- src/log_analyser/log_analyser.py | 16 ++++++++-------- src/log_analyser/objects/character.py | 4 ++-- src/log_analyser/objects/match.py | 6 +++--- src/log_analyser/objects/object.py | 4 ++-- src/log_analyser/objects/player.py | 4 ++-- src/log_analyser/objects/round.py | 4 ++-- src/log_analyser/objects/team.py | 5 +++-- src/main.py | 6 +++++- 9 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/kafka_lib.py b/src/kafka_lib.py index a449a75..e2fc014 100644 --- a/src/kafka_lib.py +++ b/src/kafka_lib.py @@ -18,8 +18,6 @@ def __init__(self, url, date=None, nb_try_connection=10): self.date = date self.nb_try_connection = nb_try_connection - - self.init_producer_ok, self.producer = self.init_producer(url) self.queue_sender = queue.Queue(2) diff --git a/src/log_analyser/log_analyser.py b/src/log_analyser/log_analyser.py index 512c011..792002e 100644 --- a/src/log_analyser/log_analyser.py +++ b/src/log_analyser/log_analyser.py @@ -1,6 +1,6 @@ import os from datetime import datetime, timedelta -from objects.match import Match +from log_analyser.objects.match import Match class LogAnalyser: @@ -29,8 +29,8 @@ def run(self): if type in self.actions: self.actions[type](line_split) - with open("../logs_process/{}.json".format(self.name.split(".")[0]), "w") as file: - file.write(self.match.export_json()) + # with open("../logs_process/{}.json".format(self.name.split(".")[0]), "w") as file: + # file.write(self.match.export_json()) def name2datetime(self): @@ -87,11 +87,11 @@ def convert_timefile_to_datetime(self, time_string): return duration -for file in os.listdir("../logs"): - if file.endswith(".txt"): - print(file) - la = LogAnalyser('../logs/{}'.format(file), file) - la.run() +# for file in os.listdir("../logs"): +# if file.endswith(".txt"): +# print(file) +# la = LogAnalyser('../logs/{}'.format(file), file) +# la.run() # la = LogAnalyser('../logs/Log-2023-12-22-21-12-32.txt', "Log-2023-12-22-21-12-32.txt") # la.run() \ No newline at end of file diff --git a/src/log_analyser/objects/character.py b/src/log_analyser/objects/character.py index 62e251a..d8742ea 100644 --- a/src/log_analyser/objects/character.py +++ b/src/log_analyser/objects/character.py @@ -1,4 +1,4 @@ -from objects.object import Object +from log_analyser.objects.object import Object class Character(Object): @@ -22,7 +22,7 @@ def add_kill(self, data): self.kills.append(data) def add_death(self, data): - self.deads.append(data) + self.deaths.append(data) def add_offensive_assist(self, data): self.offensive_assists.append(data) diff --git a/src/log_analyser/objects/match.py b/src/log_analyser/objects/match.py index cda2f79..2e126be 100644 --- a/src/log_analyser/objects/match.py +++ b/src/log_analyser/objects/match.py @@ -1,6 +1,6 @@ -from objects.object import Object -from objects.round import Round -from objects.team import Team +from log_analyser.objects.object import Object +from log_analyser.objects.round import Round +from log_analyser.objects.team import Team from datetime import datetime diff --git a/src/log_analyser/objects/object.py b/src/log_analyser/objects/object.py index 987f975..2ef2f2e 100644 --- a/src/log_analyser/objects/object.py +++ b/src/log_analyser/objects/object.py @@ -55,8 +55,8 @@ def export_json(self): dict_class_final.pop(key) else: dict_class_final[key] = self.export_json_recursive(value) - return json.dumps(dict_class_final, indent=4, sort_keys=True, default=str) - + # return json.dumps(dict_class_final, indent=4, sort_keys=True, default=str) + return json.dumps(dict_class_final, skipkeys=False, ensure_ascii=True, check_circular=True) def convert_timefile_to_datetime(self, time_string): diff --git a/src/log_analyser/objects/player.py b/src/log_analyser/objects/player.py index c6266fa..c8277a1 100644 --- a/src/log_analyser/objects/player.py +++ b/src/log_analyser/objects/player.py @@ -1,6 +1,6 @@ # from src.log_analyser.character import Character -from objects.object import Object -from objects.character import Character +from log_analyser.objects.object import Object +from log_analyser.objects.character import Character class Player(Object): diff --git a/src/log_analyser/objects/round.py b/src/log_analyser/objects/round.py index f08062a..4ab375e 100644 --- a/src/log_analyser/objects/round.py +++ b/src/log_analyser/objects/round.py @@ -1,5 +1,5 @@ -from objects.object import Object -from objects.team import Team +from log_analyser.objects.object import Object +from log_analyser.objects.team import Team class Round(Object): diff --git a/src/log_analyser/objects/team.py b/src/log_analyser/objects/team.py index e0f022c..20db965 100644 --- a/src/log_analyser/objects/team.py +++ b/src/log_analyser/objects/team.py @@ -1,5 +1,6 @@ -from objects.object import Object -from objects.player import Player +from log_analyser.objects.object import Object +from log_analyser.objects.player import Player + class Team(Object): diff --git a/src/main.py b/src/main.py index 6f66a25..a7545a3 100644 --- a/src/main.py +++ b/src/main.py @@ -2,6 +2,7 @@ import time import sys from kafka_lib import ProducerThread, ConsumerThread +from log_analyser.log_analyser import LogAnalyser class DatastrikePythonProcessing: @@ -21,7 +22,10 @@ def __init__(self): def on_callback_test(self, topic, data): print("message receive : ", topic, data) - self.producer_thread.send("analyse_report", "test") + la = LogAnalyser('logs/Log-2023-12-22-21-12-32.txt', "Log-2023-12-22-21-12-32.txt") + la.run() + a = la.match.export_json() + self.producer_thread.send("analyse.report", la.match.export_json()) def run(self): From 6428866a51ced75b36430e8ecd4660c2ef4e9991 Mon Sep 17 00:00:00 2001 From: benjamin Date: Wed, 10 Jan 2024 12:23:05 +0100 Subject: [PATCH 5/5] refactor de Matche en Map --- src/log_analyser/log_analyser.py | 38 +++++++++---------- src/log_analyser/objects/{match.py => map.py} | 10 ++--- src/log_analyser/objects/object.py | 4 +- src/main.py | 5 +-- 4 files changed, 28 insertions(+), 29 deletions(-) rename src/log_analyser/objects/{match.py => map.py} (96%) diff --git a/src/log_analyser/log_analyser.py b/src/log_analyser/log_analyser.py index 792002e..548d037 100644 --- a/src/log_analyser/log_analyser.py +++ b/src/log_analyser/log_analyser.py @@ -1,6 +1,6 @@ import os from datetime import datetime, timedelta -from log_analyser.objects.match import Match +from log_analyser.objects.map import Map class LogAnalyser: @@ -12,9 +12,9 @@ def __init__(self, path_csv, name) -> None: self.date = self.name2datetime() - self.match = None + self.map = None - self.actions = {"match_start": self.process_match_start} + self.actions = {"match_start": self.process_map_start} def run(self): @@ -30,7 +30,7 @@ def run(self): self.actions[type](line_split) # with open("../logs_process/{}.json".format(self.name.split(".")[0]), "w") as file: - # file.write(self.match.export_json()) + # file.write(self.map.export_json()) def name2datetime(self): @@ -40,9 +40,9 @@ def name2datetime(self): return date_object - def process_match_start(self, data): + def process_map_start(self, data): - self.match = Match.from_json({"rounds": [], + self.map = Map.from_json({"rounds": [], "date": self.date, "map_name": data[3], "map_type": data[4], @@ -52,30 +52,30 @@ def process_match_start(self, data): "score_team2": 0, }) - self.actions = {"match_start": self.process_match_start, - "round_start": self.match.add_round, - "round_end": self.match.end_round, + self.actions = {"match_start": self.process_map_start, + "round_start": self.map.add_round, + "round_end": self.map.end_round, "hero_spawn": self.process_hero_spawn, "hero_swap": self.process_hero_swap, - "kill": self.match.add_kill, - "ultimate_charged": self.match.add_ultimate_charged, - "ultimate_start": self.match.add_ultimate_start, - "ultimate_end": self.match.add_ultimate_end, - "objective_captured": self.match.add_objective_captured, - "player_stat": self.match.add_player_stat, - "point_progress": self.match.add_objective_progress, - "payload_progress": self.match.add_objective_progress, + "kill": self.map.add_kill, + "ultimate_charged": self.map.add_ultimate_charged, + "ultimate_start": self.map.add_ultimate_start, + "ultimate_end": self.map.add_ultimate_end, + "objective_captured": self.map.add_objective_captured, + "player_stat": self.map.add_player_stat, + "point_progress": self.map.add_objective_progress, + "payload_progress": self.map.add_objective_progress, } def process_hero_spawn(self, data): player_data = {"time": data[2], "team_name": data[3], "player_name": data[4], "character_name": data[5]} - self.match.add_player(player_data) + self.map.add_player(player_data) def process_hero_swap(self, data): hero_data = {"time": data[2], "team_name": data[3], "player_name": data[4], "character_name": data[5], "character_swap": data[6]} - self.match.add_hero_swap(hero_data) + self.map.add_hero_swap(hero_data) def convert_timefile_to_datetime(self, time_string): diff --git a/src/log_analyser/objects/match.py b/src/log_analyser/objects/map.py similarity index 96% rename from src/log_analyser/objects/match.py rename to src/log_analyser/objects/map.py index 2e126be..c0ac064 100644 --- a/src/log_analyser/objects/match.py +++ b/src/log_analyser/objects/map.py @@ -4,7 +4,7 @@ from datetime import datetime -class Match(Object): +class Map(Object): def __init__(self, **kwargs): @@ -152,9 +152,9 @@ def end_round(self, data): self.score_team2 = end_round_data["score_team2"] print("###### END ROUND {} #######\n".format(self.actual_round)) - def end_match(self, data): + def end_map(self, data): - end_match_data = {"time": data[2], "score_team1": data[4], "score_team2": data[5]} + end_map_data = {"time": data[2], "score_team1": data[4], "score_team2": data[5]} - self.score_team1 = end_match_data["score_team1"] - self.score_team2 = end_match_data["score_team2"] + self.score_team1 = end_map_data["score_team1"] + self.score_team2 = end_map_data["score_team2"] diff --git a/src/log_analyser/objects/object.py b/src/log_analyser/objects/object.py index 2ef2f2e..1b3e6ca 100644 --- a/src/log_analyser/objects/object.py +++ b/src/log_analyser/objects/object.py @@ -31,7 +31,7 @@ def export_json_recursive(self, data): dict_class_final = dict_class.copy() for key, value in dict_class.items(): - if not key in data.class_model: + if not key in data.class_model and key != "class_name": dict_class_final.pop(key) return data.export_json_recursive(dict_class_final) @@ -51,7 +51,7 @@ def export_json(self): dict_class_final = dict_class.copy() for key, value in dict_class.items(): - if not key in self.class_model: + if not key in self.class_model and key != "class_name": dict_class_final.pop(key) else: dict_class_final[key] = self.export_json_recursive(value) diff --git a/src/main.py b/src/main.py index a7545a3..9d96bca 100644 --- a/src/main.py +++ b/src/main.py @@ -24,13 +24,12 @@ def on_callback_test(self, topic, data): la = LogAnalyser('logs/Log-2023-12-22-21-12-32.txt', "Log-2023-12-22-21-12-32.txt") la.run() - a = la.match.export_json() - self.producer_thread.send("analyse.report", la.match.export_json()) + self.producer_thread.send("analyse.report", la.map.export_json()) def run(self): while self.running: - print("Service en cours d'exécution...") + # print("Service en cours d'exécution...") time.sleep(1) print("Service stop.")