Add Large Dataset to Benchmarks (#106)

* Fixes philihp/openskill.js#453 and adds a better benchmark for ranks.

Signed-off-by: Vivek Joshy <8206808+vivekjoshy@users.noreply.github.com>

* Fix processor not found error

Signed-off-by: Vivek Joshy <8206808+vivekjoshy@users.noreply.github.com>

* Add changelog

Signed-off-by: Vivek Joshy <8206808+vivekjoshy@users.noreply.github.com>

---------

Signed-off-by: Vivek Joshy <8206808+vivekjoshy@users.noreply.github.com>

.gitattributes

@@ -1,2 +1,3 @@
 *.csv filter=lfs diff=lfs merge=lfs -text
 *.jsonl filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -133,3 +133,6 @@ dmypy.json
 
 # PDM
 /.pdm-build/
+
+# Data Files
+/benchmark/data/pubg.csv

benchmark/README.md

@@ -0,0 +1,12 @@
+# Benchmark Instructions
+
+Simply run ``benchmark.py`` with a compatible Python version and choose the options.
+
+## Available Benchmarks
+
+- ``Win``: Compares win performance against TrueSkill.
+- ``Draw``: Predicts draws on standard chess matches.
+- ``Rank``: Predicts the rank of players.
+- ``Large``: Uses rank prediction on a large multi-faction dataset.
+
+The ``Large`` benchmark requires the ``pubg.csv`` file (around 1 GB) be extracted from ``pubg.7z`` and place in the ``data`` folder.

benchmark/benchmark.py

@@ -1,3 +1,4 @@
+from processors import Draw, Large, Rank, Win
 from prompt_toolkit import HTML
 from prompt_toolkit import print_formatted_text as print
 from prompt_toolkit import prompt
@@ -11,7 +12,6 @@
     ThurstoneMostellerFull,
     ThurstoneMostellerPart,
 )
-from processors import Draw, Rank, Win
 
 
 class NumberValidator(Validator):
@@ -39,7 +39,7 @@ def validate(self, document):
 model_names = {m.__name__: m for m in models}
 model_completer = WordCompleter(list(model_names.keys()))
 
-benchmark_types = [Win, Draw, Rank]
+benchmark_types = [Win, Draw, Rank, Large]
 benchmark_type_names = {_.__name__: _ for _ in benchmark_types}
 benchmark_types_completer = WordCompleter(list(benchmark_type_names.keys()))
 
@@ -92,6 +92,15 @@ def validate(self, document):
         )
         rank_processor.process()
         rank_processor.print_result()
-else:
-    print(HTML("<style fg='Red'>Processor Not Found</style>"))
-    quit()
+    elif input_benchmark_type == "Large":
+        large_processor = Large(
+            path="data/pubg.csv",
+            seed=input_seed,
+            minimum_matches=minimum_matches,
+            model=model,
+        )
+        large_processor.process()
+        large_processor.print_result()
+    else:
+        print(HTML("<style fg='Red'>Processor Not Found</style>"))
+        quit()

benchmark/processors/__init__.py

@@ -1,3 +1,4 @@
 from .draw import Draw
+from .large import Large
 from .rank import Rank
 from .win import Win