update notebook

examples/benchmarks/analysis.py

@@ -25,7 +25,7 @@
 client = Client()
 dset = Dataset.create(name="bird-identification")
 model = Model.create(name="some_model")
-PAIR_LIMIT = 10
+PAIR_LIMIT = 500
 
 
 def time_func(func):
@@ -42,20 +42,22 @@ def wrap_func(*args, **kwargs):
 def write_results_to_file(write_path: str, result_dict: dict):
     current_datetime = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
 
-    with open(write_path, "a+") as f:
-        try:
-            result = json.load(f)
-        except json.JSONDecodeError:
-            result = {}
-        result[current_datetime] = result_dict
-        f.write(json.dumps(result))
+    with open(write_path, "a+") as file:
+        file.seek(0)
+        data = json.load(file)
 
+    data[current_datetime] = result_dict
 
-def ingest_groundtruths_and_predictions(raw: dict):
-    number_of_pairs = len(raw["groundtruth_prediction_pairs"])
+    with open(write_path, "w+") as file:
+        json.dump(data, file, indent=4)
+
+
+def ingest_groundtruths_and_predictions(raw: dict, pair_limit: int):
     groundtruths = []
     predictions = []
-    for groundtruth, prediction in raw["groundtruth_prediction_pairs"]:
+    for groundtruth, prediction in raw["groundtruth_prediction_pairs"][
+        :pair_limit
+    ]:
         groundtruths.append(
             GroundTruth(
                 datum=Datum(
@@ -100,11 +102,6 @@ def ingest_groundtruths_and_predictions(raw: dict):
             )
         )
 
-    assert (
-        len(predictions) == number_of_pairs
-        and len(groundtruths) == number_of_pairs
-    )
-
     for gt in groundtruths:
         dset.add_groundtruth(gt)
 
@@ -142,7 +139,7 @@ def time_functions():
 
     start_time = time()
 
-    ingest_groundtruths_and_predictions(raw=raw)
+    ingest_groundtruths_and_predictions(raw=raw, pair_limit=PAIR_LIMIT)
     ingest_time = f"{(time() - start_time):.4f}"
 
     run_base_evaluation(dset=dset, model=model)

examples/benchmarks/old_results.json

@@ -0,0 +1 @@
+{"25/06/2024 14:19:11": {"limit": 10, "ingest": "0.7415", "base": "1.9897", "pr": "4.2648"}}{"25/06/2024 14:28:04": {"limit": 100, "ingest": "7.3905", "base": "42.1022", "pr": "185.7983"}}

examples/benchmarks/results.json

@@ -1 +1,17 @@
-{"25/06/2024 13:35:17": {"a": "a"}}
+{
+    "25/06/2024 15:10:21": {
+        "limit": "blah"
+    },
+    "25/06/2024 15:10:36": {
+        "limit": "blah"
+    },
+    "25/06/2024 15:10:40": {
+        "limit": "blah"
+    },
+    "25/06/2024 15:11:42": {
+        "limit": 10,
+        "ingest": "1.0709",
+        "base": "2.3355",
+        "pr": "4.6418"
+    }
+}