2023-04 Model

.gitignore

@@ -16,7 +16,6 @@ data/**/*.pkl
 *.synctex.gz
 *.osr
 lightning_logs/
-models/
 .streamlit/
 poetry.lock
 .data/

README.md

@@ -7,7 +7,7 @@ It uses Neural Collaborative Filtering to learn associations between user and ma
 predict new scores never before seen.
 
 **Performance Error Graph**
-![Performance Graph](models/V2_2023_01/error.png)
+![Performance Graph](models/V2_2023_04/error.png)
 
 ## :hourglass_flowing_sand: Project Status
 
@@ -35,7 +35,7 @@ If you want to use this locally, you need Python and the packages listed in [req
 
 Below is a recipe on how to use it.
 ```py
-path_to_model = MODEL_DIR / "V2_2023_01/checkpoints/epoch=5-step=43584.ckpt"
+path_to_model = "oath/to/model.ckpt"
 net = NeuMF.load_from_checkpoint(path_to_model.as_posix())
 
 # THIS MUST BE RAN TO AVOID TRAINING THE MODEL

app/Main.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 
 import sys
+import torch
 from google.cloud import firestore
 
 sys.path.append(Path(__file__).parents[1].as_posix())
@@ -45,7 +46,7 @@ def add_analytics_count(add: int):
 
 
 @st.cache_resource
-def get_model(model_path=MODEL_DIR / "V2_2023_01/checkpoints/epoch=5-step=43584.ckpt"):
+def get_model(model_path=MODEL_DIR / "V2_2023_04/checkpoints/epoch=8-step=55773.ckpt"):
     net = NeuMF.load_from_checkpoint(model_path.as_posix(), map_location=torch.device('cpu'))
     net.eval()
     return net
@@ -107,17 +108,17 @@ def get_beatmap_metadata(beatmap_id: int):
 with st.sidebar:
     st.header(":game_die: Metrics")
     st.markdown("""
-    ![R2](https://img.shields.io/badge/R%20Squared-81.48%25-blueviolet)
-    ![MAE](https://img.shields.io/badge/MAE-1.18%25-blue)
-    ![RMSE](https://img.shields.io/badge/RMSE-1.71%25-blue)
+    ![R2](https://img.shields.io/badge/R%20Squared-71.88%25-blueviolet)
+    ![MAE](https://img.shields.io/badge/MAE-1.14%25-blue)
+    ![RMSE](https://img.shields.io/badge/RMSE-1.68%25-blue)
     ![Model Size](https://img.shields.io/github/size/Eve-ning/opal/models/V2_2023_01/checkpoints/epoch%253D5-step%253D43584.ckpt?color=purple&label=Model%20Size&logo=pytorch-lightning)
     """)
     st.header(":bookmark: Requirements")
     st.markdown("""
     1) Only osu!mania.
 
     The user must be:
-    1) ranked <10K in 1st Jan 2023
+    1) ranked <10K in 1st Apr 2023
     2) active in that predicted year
 
     The map must be:
@@ -199,9 +200,9 @@ def get_beatmap_metadata(beatmap_id: int):
 pred_nt = df_pred_last.loc[(df_pred_last['speed'] == 'NT'), 'pred']
 pred_dt = df_pred_last.loc[(df_pred_last['speed'] == 'DT'), 'pred']
 
-pred_ht = float(pred_ht) if pred_ht.any() else None
-pred_nt = float(pred_nt) if pred_nt.any() else None
-pred_dt = float(pred_dt) if pred_dt.any() else None
+pred_ht = float(pred_ht.iloc[0]) if pred_ht.any() else None
+pred_nt = float(pred_nt.iloc[0]) if pred_nt.any() else None
+pred_dt = float(pred_dt.iloc[0]) if pred_dt.any() else None
 
 c1, c2, c3 = st.columns(3)
 

models/README.md

@@ -2,9 +2,10 @@
 
 **See below on how to load the model.**
 
-| Model      | R2     | MAE    | RMSE  | Error Distribution             |
-|------------|--------|--------|-------|--------------------------------|
+| Model      | R2     | MAE   | RMSE  | Error Distribution             |
+|------------|--------|-------|-------|--------------------------------|
 | V2_2023_01 | 81.48% | 1.18% | 1.71% | ![Error](V2_2023_01/error.png) |
+| V2_2023_04 | 71.88% | 1.14% | 1.68% | ![Error](V2_2023_04/error.png) |
 
 **Note: V1 is deprecated due to incompatibility with new model signature.**
 ## Limitations

opal/score/datamodule/score_datamodule.py

@@ -115,6 +115,8 @@ def prep_metric(df: pd.DataFrame,
             Adds Year column
         """
 
+        df = df.dropna()
+        df['enabled_mods'] = df['enabled_mods'].astype(int)
         half_score = (
                 ((df['enabled_mods'] & OsuMod.EASY) > 0) |
                 ((df['enabled_mods'] & OsuMod.NO_FAIL) > 0) |

opal/train.py

@@ -1,6 +1,6 @@
 import pytorch_lightning as pl
 import torch
-from pytorch_lightning.callbacks import LearningRateMonitor, EarlyStopping, ModelCheckpoint, ModelPruning, QuantizationAwareTraining
+from pytorch_lightning.callbacks import LearningRateMonitor, EarlyStopping, ModelCheckpoint
 
 from opal.score.collaborative_filtering import NeuMF
 from opal.score.datamodule import ScoreDataModule
@@ -41,12 +41,11 @@ def train(yyyy_mm: str):
             LearningRateMonitor(),
             ModelCheckpoint(monitor='val_loss', save_top_k=1, mode='min')
         ],
-        devices=[3, ],
     )
 
     trainer.fit(net, datamodule=dm)
 
 
 if __name__ == '__main__':
     torch.set_float32_matmul_precision('high')
-    train("2023_01")
+    train("2023_04")