Merge pull request #81 from chorus-ai/add-cloud-performance

Add cloud hardware benchmarking

tests/cloud_performance/aggregate_results.py

@@ -0,0 +1,41 @@
+import pandas as pd
+import glob
+
+size_list = []
+record_list = []
+time_list = []
+mode_list = []
+format_list = []
+
+for filepath in glob.glob('/path/to/results/*/*/*.txt'):
+    with open(filepath, "r") as f:
+        start = False
+        sz = filepath.split('/')[4].split('-')[1]
+        record = filepath.split('/')[5]
+        format = filepath.split('/')[6].replace('benchmark_results_', '').replace('.txt', '')
+        for line in f:
+            if start and '___' in line:
+                start = False
+            if start:
+                size_list = size_list + [sz]
+                record_list = record_list + [record]
+                format_list = format_list + [format]
+                str_list = list(filter(None, line.split('  ')))
+                if len(str_list) == 5:
+                    time_list = time_list + [str_list[3]]
+                    mode_list = mode_list + [str_list[4].replace('\n', '')]
+                else:
+                    time_list = time_list + ['N/A']
+                    mode_list = mode_list + ['N/A']
+            if "#seek" in line:
+                start = True
+
+
+dict = {'size': size_list,
+        'record': record_list,
+        'format': format_list,
+        'time': time_list,
+        'mode': mode_list}
+df = pd.DataFrame(dict)
+print(df.head())
+df.to_csv('performance_processed.csv')

tests/cloud_performance/files.txt

@@ -0,0 +1,8 @@
+waveform_benchmark/formats/dicom.py
+waveform_benchmark/formats/wfdb.py
+waveform_benchmark/formats/base.py
+waveform_benchmark/formats/ccdef.py
+waveform_benchmark/formats/zarr.py
+waveform_benchmark/formats/npy.py
+waveform_benchmark/formats/parquet.py
+waveform_benchmark/formats/pickle.py

tests/cloud_performance/records.txt

@@ -0,0 +1,10 @@
+/waveform/waveforms/challenge-2018/1.0.0/training/tr03-0079/tr03-0079
+/waveform/waveforms/challenge-2018/1.0.0/training/tr14-0272/tr14-0272
+/waveform/waveforms/charisdb/1.0.0/charis1
+/waveform/waveforms/charisdb/1.0.0/charis5
+/waveform/waveforms/GE_WFDB/A004-0502604815/A004-0502604815
+/waveform/waveforms/GE_WFDB/A004-0503168997/A004-0503168997
+/waveform/waveforms/mimic3wdb/1.0/36/3654093/3654093
+/waveform/waveforms/mimic3wdb/1.0/37/3740336/3740336
+/waveform/waveforms/mimic4wdb/0.1.0/waves/p100/p10079700/85594648/85594648
+/waveform/waveforms/mimic4wdb/0.1.0/waves/p169/p16955095/82284982/82284982

tests/cloud_performance/run_tests.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+RECORDS=$(cat records.txt)
+FILES=$(cat files.txt)
+SIZE=xs #CHANGEME
+
+while read record;
+do
+  while read file;
+  do
+    # Skip base.py file
+    if [[ "$file" == *"base.py" ]]; then
+    echo "Skipping benchmark for $file"
+    continue
+    fi
+
+    # Extract the module path from the file path
+    MODULE_PATH=$(echo "$file" | sed 's/\//./g' | sed 's/\.py$//')
+
+    # Find and list all classes in the module
+    echo "Running benchmarks for classes in $MODULE_PATH"
+    CLASSES=$(python3 -c "import sys; import inspect; from importlib import import_module; mod = import_module('$MODULE_PATH'); print(' '.join(cls for cls, obj in inspect.getmembers(mod, inspect.isclass) if obj.__module__ == mod.__name__ and not cls.startswith('Base')))")
+    IFS=' ' read -ra CLS <<< "$CLASSES"
+
+    for cls in "${CLS[@]}"
+    do
+      CLASS_PATH="$MODULE_PATH.$cls"
+      FILENAME="benchmark_results_${cls}.txt"
+      IFS='/' read -r -a array <<< "$record"
+      FILEDIR=$(echo "${array[-1]}")
+      mkdir -p "/waveform/results-$SIZE/$FILEDIR"
+      echo "Benchmarking $CLASS_PATH for $FILEDIR"
+      ./waveform_benchmark.py -r "$record" -f "$CLASS_PATH" > "/waveform/results-$SIZE/$FILEDIR/$FILENAME"
+    done
+  done < files.txt
+done < records.txt