Shot trainer

examples/base_keras_model.py

@@ -69,7 +69,6 @@ def write_log(self, callback, names, logs, batch_no, eval=False):
 
     def finalize_model(self, logname=str(int(random() * 1000))):
 
-
         loss, loss_weights = self.create_loss()
         self.model.compile(tf.keras.optimizers.Nadam(lr=0.001), loss=loss, loss_weights=loss_weights,
                            metrics=[tf.keras.metrics.mean_absolute_error, tf.keras.metrics.binary_accuracy])

examples/replays/shots/shot_input_formatter.py

@@ -0,0 +1,43 @@
+import pandas as pd
+import numpy as np
+from framework.input_formatter.base_input_formatter import BaseInputFormatter
+from framework.replay.replay_format import GeneratedHit
+
+
+class ShotInputFormatter(BaseInputFormatter):
+    def get_input_state_dimension(self):
+        pass
+
+    def create_input_array(self, input_data: GeneratedHit, batch_size=1):
+        result = []
+        hit = input_data.get_hit()
+        frame = hit.frame_number
+        df = input_data.get_replay().get_pandas()
+        index = df.index
+        hit_frame = df.loc[frame]
+
+        new_frame = hit_frame['ball']
+
+        proto = input_data.get_replay().get_proto()
+        hit_player = hit.player_id
+        blue_team = []
+        orange_team = []
+        for player in proto.players:
+            if player.is_orange:
+                orange_team.append(player)
+            else:
+                blue_team.append(player)
+
+        return result
+
+    def get_speed(self, frame):
+        return np.sqrt(frame['vel_x']**2 + frame['vel_y']**2 + frame['vel_z']**2)
+
+    def get_distance_from_goal(self, frame, player, team):
+        if team == 0:
+            return np.sqrt(frame['pos_x']**2 + (frame['pos_y'] - (6000 * (1 - team)))**2 + frame['vel_z']**2)
+
+    def get_player_data(self, frame):
+        return [frame['pos_x'], frame['pos_y'], frame['pos_z'],
+                frame['rot_x'], frame['rot_y'], frame['rot_z'],
+                frame['vel_x'], frame['vel_y'], frame['vel_z']]

examples/replays/shots/shot_model.py

@@ -0,0 +1,17 @@
+from tensorflow.python.keras import Model
+
+from examples.base_keras_model import BaseKerasModel
+from framework.output_formatter.base_output_formatter import BaseOutputFormatter
+import tensorflow as tf
+
+
+class ShotModel(BaseKerasModel):
+
+    def create_output_layer(self, output_formatter: BaseOutputFormatter, hidden_layer=None):
+        # sigmoid/tanh all you want on self.model
+        if hidden_layer is None:
+            hidden_layer = self.hidden_layer
+        self.outputs = tf.keras.layers.Dense(output_formatter.get_model_output_dimension()[0],
+                                             activation='sigmoid')(hidden_layer)
+        self.model = Model(inputs=self.inputs, outputs=self.outputs)
+        return self.outputs

examples/replays/shots/shot_output_formatter.py

@@ -0,0 +1,11 @@
+from framework.output_formatter.base_output_formatter import BaseOutputFormatter
+from framework.replay.replay_format import GeneratedHit
+
+
+class ShotOutputFormatter(BaseOutputFormatter):
+
+    def get_model_output_dimension(self):
+        return [1]
+
+    def create_array_for_training(self, input_hit: GeneratedHit, batch_size=1):
+        return [int(input_hit.get_hit().goal)]

examples/replays/shots/shot_trainer.py

@@ -0,0 +1,13 @@
+from examples.replays.shots.shot_input_formatter import ShotInputFormatter
+from examples.replays.shots.shot_output_formatter import ShotOutputFormatter
+from framework.data_generator.local_cache_creator import LocalCacheCreator
+from framework.data_generator.replay.hit_generator import HitGenerator
+
+if __name__ == '__main__':
+    hit_generator = HitGenerator()
+    hit_generator.initialize(hit_filter={'shot': True})
+    cache = LocalCacheCreator(ShotInputFormatter(), ShotOutputFormatter(), hit_generator)
+
+    cache.create_cache()
+
+    cache.save_cache('cache.ch')

framework/data_generator/local_cache_creator.py

@@ -0,0 +1,31 @@
+import numpy as np
+import pandas as pd
+from carball.analysis.utils.pandas_manager import PandasManager
+
+from framework.data_generator.base_generator import BaseDataGenerator
+from framework.input_formatter.base_input_formatter import BaseInputFormatter
+from framework.output_formatter.base_output_formatter import BaseOutputFormatter
+
+
+class LocalCacheCreator:
+
+    def __init__(self, input_formatter: BaseInputFormatter, output_formatter: BaseOutputFormatter, data_generator: BaseDataGenerator):
+        self.data_generator = data_generator
+        self.output_formatter = output_formatter
+        self.input_formatter = input_formatter
+        self.cache = None
+
+    def create_cache(self):
+        input_array = np.array([])
+        output_array = np.array([])
+
+        for data in self.data_generator.get_data():
+            np.append(input_array, np.array(self.input_formatter.create_input_array(data)))
+            np.append(output_array, np.array(self.output_formatter.create_array_for_training(data)))
+
+        self.cache = pd.DataFrame(data={"input": input_array, "output": output_array})
+
+    def save_cache(self, file_path):
+        result = PandasManager.safe_write_pandas_to_memory(self.cache)
+        with open(file_path, 'w') as f:
+            f.write(result)

framework/data_generator/replay/replay_generator.py

@@ -1,3 +1,4 @@
+import io
 import random
 
 import requests
@@ -6,6 +7,11 @@
 from framework.replay.replay_format import GeneratedReplay
 
 
+def create_in_memory_file(response: bytes) -> io.BytesIO:
+    in_memory_file = io.BytesIO(response)
+    return in_memory_file
+
+
 class ReplayListGenerator(BaseDataGenerator):
 
     BASE_URL = "https://calculated.gg"
@@ -20,11 +26,11 @@ def __init__(self, api_key=1, min_mmr=0, max_mmr=4000, num_players_on_team=-1, m
 
         self.shuffle = shuffle
         self.next_page = True
-        self.existing_url = '/api/v1/replays?page=1&key=' + str(api_key)
+        self.existing_url = None
         self.replays = []
 
-    def initialize(self, **kwargs):
-        pass
+    def initialize(self, initial_page=1):
+        self.existing_url = '/api/v1/replays?page=' + str(initial_page) + '&key=' + str(self.api_key)
 
     def create_url(self, existing_url):
         return self.BASE_URL + existing_url +'&minmmr=' + str(self.min_mmr) + '&max_mmr=' + str(self.max_mmr)
@@ -86,7 +92,8 @@ def download_replay(self, replay_hash):
         # get pts
         pts = requests.get(self.BASE_URL + self.DOWNLOAD_URL + replay_hash + '.replay.pts' + self.key_url)
         pandas = requests.get(self.BASE_URL + self.DOWNLOAD_URL + replay_hash + '.replay.gzip' + self.key_url)
-        replay = GeneratedReplay(protobuf=pts.content, pandas=pandas.content)
+
+        replay = GeneratedReplay(protobuf=pts.content, pandas=create_in_memory_file(pandas.content))
         return replay
 
     def __get_next_replay(self):

framework/replay/replay_format.py

@@ -1,3 +1,4 @@
+import gzip
 import zlib
 
 from carball.analysis.utils.pandas_manager import PandasManager
@@ -37,7 +38,15 @@ def get_pandas(self) -> pd.DataFrame:
         """
         if self.decoded_pandas is not None:
             return self.decoded_pandas
-        self.decoded_pandas = PandasManager.safe_read_pandas_to_memory(zlib.decompress(self.pandas))
+        self.pandas.seek(0)
+        with gzip.GzipFile(fileobj=self.pandas, mode='rb') as f:
+            self.decoded_pandas = PandasManager.safe_read_pandas_to_memory(f)
+
+        self.decoded_pandas.set_index('index', drop=True, inplace=True)
+        columns = []
+        for tuple_str in self.decoded_pandas.columns.values:
+            columns.append(eval(tuple_str))
+        self.decoded_pandas.columns = pd.MultiIndex.from_tuples(columns)
         self.pandas = None
         return self.decoded_pandas
 

trainer/base_trainer.py → framework/trainer/base_trainer.py

@@ -6,3 +6,9 @@ class BaseTrainer:
     def __init__(self, model_holder: BaseModelHolder):
         self.model_holder = model_holder
         model_holder.initialize_model(load=True)
+
+    def initialize_training(self):
+        raise NotImplementedError()
+
+    def train(self):
+        raise NotImplementedError()

framework/trainer/generated_data_trainer.py

@@ -0,0 +1,17 @@
+from framework.data_generator.base_generator import BaseDataGenerator
+from framework.model_holder.base_model_holder import BaseModelHolder
+from framework.trainer.base_trainer import BaseTrainer
+
+
+class GeneratedDataTrainer(BaseTrainer):
+
+    def __init__(self, model_holder: BaseModelHolder, data_generator: BaseDataGenerator):
+        super().__init__(model_holder)
+        self.data_generator = data_generator
+
+    def initialize_training(self, **kwargs):
+        self.data_generator.initialize(**kwargs)
+
+    def train(self):
+        for data in self.data_generator.get_data():
+            self.model_holder.train_step(data, data)