Merge #6238

6238: Update lsu perftest baseline r=hkaiser a=Pansysk75



Co-authored-by: Panos Syskakis <pansysk75@gmail.com>

.jenkins/lsu-perftests/launch_perftests.sh

@@ -16,13 +16,24 @@ hpx_targets=(
 hpx_test_options=(
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --hpx:threads=4 --vector_size=104857 --work_delay=1 \
-    --chunk_size=0 --test_count=5000"
+    --chunk_size=0 --test_count=200"
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --hpx:queuing=local-priority --hpx:threads=4 --test-all \
-    --repetitions=500 --futures=207270"
+    --repetitions=40 --futures=207270"
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
-    --vector_size=518176 --hpx:threads=4 --iterations=5000 \
-    --warmup_iterations=500")
+    --vector_size=518176 --hpx:threads=4 --iterations=200 \
+    --warmup_iterations=20")
+
+
+    #     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
+    # --hpx:threads=4 --vector_size=10000 --work_delay=1 \
+    # --chunk_size=0 --test_count=5000"
+    # "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
+    # --hpx:queuing=local-priority --hpx:threads=4 --test-all \
+    # --repetitions=100 --futures=500000"
+    # "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
+    # --vector_size=1048576 --hpx:threads=4 --iterations=5000 \
+    # --warmup_iterations=500")
 
 # Build binaries for performance tests
 ${perftests_dir}/driver.py -v -l $logfile build -b release -o build \
@@ -37,10 +48,13 @@ ${perftests_dir}/driver.py -v -l $logfile build -b release -o build \
 index=0
 result_files=""
 
+n_executions=50
+
 # Run and compare for each targets specified
 for executable in "${hpx_targets[@]}"; do
     test_opts=${hpx_test_options[$index]}
     result=${build_dir}/reports/${executable}.json
+
     reference=${perftests_dir}/perftest/references/lsu_default/${executable}.json
     result_files+=(${result})
     references_files+=(${reference})
@@ -52,7 +66,7 @@ for executable in "${hpx_targets[@]}"; do
 
     # Run performance tests
     ${perftests_dir}/driver.py -v -l $logfile_tmp perftest run --local True \
-        --run_output $result --targets-and-opts "${run_command[@]}" ||
+        --run_output $result --targets-and-opts "${run_command[@]}" --n_executions $n_executions  ||
         {
             echo 'Running failed'
             test_errors=1

tools/perftests_ci/driver.py

@@ -126,7 +126,12 @@ def perftest():
               type=str,
               help='extra arguments to pass to the test\nWarning prefer = to \
               space to assign values to hpx options')
-    def run(local, run_output, targets_and_opts):
+    @args.arg('--n_executions',
+              default=1,
+              type=int,
+              help='number of executions of the benchmark executable, whose \
+              results are then merged together')
+    def run(local, run_output, targets_and_opts, n_executions):
         # options
         targets_and_opts = ' '.join(targets_and_opts).lstrip()
 
@@ -136,7 +141,7 @@ def run(local, run_output, targets_and_opts):
         mkdirp(run_output)
 
         import perftest
-        data = perftest.run(local, targets_and_opts)
+        data = perftest.run(local, targets_and_opts, n_executions)
         with open(run_output, 'w') as outfile:
             json.dump(data, outfile, indent='  ')
             log.info(f'Successfully saved perftests output to {run_output}')

tools/perftests_ci/perftest/__init__.py

@@ -36,21 +36,35 @@ def _now():
     return datetime.now(timezone.utc).astimezone().isoformat()
 
 
-def run(local, targets_and_opts):
+def run(local, targets_and_opts, n_executions):
     from pyutils import buildinfo
 
+    def join_results(result_list):
+        # This should work, as long as the format of the result files stays consistent
+        joined_results = result_list[0]
+        for result in result_list[1:]:
+              for i in range(len(joined_results["outputs"])):
+                joined_results["outputs"][i]["series"].extend(result["outputs"][i]["series"])
+        return joined_results
+
     binary_dir = buildinfo.binary_dir
     command = []
     if targets_and_opts:
         run_command = os.path.join(binary_dir, targets_and_opts)
         command += run_command.split()
 
-    if local:
-        output = runtools.run(command)
-    else:
-        output = runtools.srun(command)
 
-    data = json.loads(output)
+    result_list = []
+
+    for _ in range(n_executions):
+        if local:
+            output = runtools.run(command)
+        else:
+            output = runtools.srun(command)
+
+        result_list.append(json.loads(output))
+
+    data = join_results(result_list)
 
     data[var._project_name] = {
         'commit': _git_commit(), 'datetime': _git_datetime()}