diff --git a/benchmarks/lockhammer/README.rst b/benchmarks/lockhammer/README.rst index 8dc30c1..88412eb 100644 --- a/benchmarks/lockhammer/README.rst +++ b/benchmarks/lockhammer/README.rst @@ -38,13 +38,52 @@ as possible). Detailed information about each run is printed to stderr while a CSV summary is printed to stdout. +Software Dependencies +--------------------- + ++ gcc ++ python3 ++ sh python3 module ++ yaml python3 module + Usage ===== The build system will generate a separate lockhammer binary for each test with the format lh_[testname]. Each lockhammer binary accepts the following options: - [-t threads] Number of threads to exercise, default online cores - [-a acquires] Number of acquisitions per thread, default 50000 - [-c critical] Critical section in loop iterations, default 0 - [-p parallel] Parallelizable section in loop iterations, default 0 - [-s] Run in safe mode, default no +:: + [-t threads] Number of threads to exercise, default online cores + [-a acquires] Number of acquisitions per thread, default 50000 + [-c critical] Critical section in loop iterations, default 0 + [-p parallel] Parallelizable section in loop iterations, default 0 + [-s] Run in safe mode with normal priority threads instead of RT_FIFO priority, default no + + +Plotting +======== + +The default plotting script utilizes jupyter-notebook, matplotlib, seaborn +and pandas under python3 environment. For Ubuntu on x86_64 machine, the +following packages have to be installed: +apt install build-essential python3 python3-pip jupyter-notebook + +For aarch64 machine, additional packages are also needed: +apt install pkg-config libfreetype6-dev python3-scipy + +Then pip3 can install all plotting related libraries with the following cmd: +pip3 install matplotlib seaborn pandas numpy + +Note, seaborn has to be installed without scipy as dependency on aarch64: +pip3 install seaborn --no-dependencies + +The jupyter-notebook can be started with: +jupyter-notebook --ip 0.0.0.0 --port=8888 + +Now any browser should be able to access the jupyter notebook called: +lockhammer-jupyter-notebook.ipynb + +Start a browser, with IP address set to the jupyter server IP and port 8888: +e.g. http://example.test.com:8888 + +Click the notebook named lockhammer-jupyter-notebook.ipynb, run each cell one +by one and jupyter should be able to generate the png graph locally. diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_event_mutex_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_event_mutex_200ns_1000ns.png new file mode 100644 index 0000000..875eefa Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_event_mutex_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_lockref_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_lockref_200ns_1000ns.png new file mode 100644 index 0000000..de97ff3 Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_lockref_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_rw_lock_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_rw_lock_200ns_1000ns.png new file mode 100644 index 0000000..2012b2f Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_cas_rw_lock_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_empty_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_empty_200ns_1000ns.png new file mode 100644 index 0000000..9a22655 Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_empty_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_event_mutex_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_event_mutex_200ns_1000ns.png new file mode 100644 index 0000000..ab372ef Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_event_mutex_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_incdec_refcount_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_incdec_refcount_200ns_1000ns.png new file mode 100644 index 0000000..467a6d9 Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_incdec_refcount_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_jvm_objectmonitor_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_jvm_objectmonitor_200ns_1000ns.png new file mode 100644 index 0000000..6de79b9 Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_jvm_objectmonitor_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_osq_lock_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_osq_lock_200ns_1000ns.png new file mode 100644 index 0000000..3de44ad Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_osq_lock_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_queued_spinlock_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_queued_spinlock_200ns_1000ns.png new file mode 100644 index 0000000..822265a Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_queued_spinlock_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_swap_mutex_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_swap_mutex_200ns_1000ns.png new file mode 100644 index 0000000..23ca8d0 Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_swap_mutex_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_tbb_spin_rw_mutex_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_tbb_spin_rw_mutex_200ns_1000ns.png new file mode 100644 index 0000000..28a1139 Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_tbb_spin_rw_mutex_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_ticket_spinlock_200ns_1000ns.png b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_ticket_spinlock_200ns_1000ns.png new file mode 100644 index 0000000..1083b34 Binary files /dev/null and b/benchmarks/lockhammer/graphs/github_lockhammer_all_common_20181106_ticket_spinlock_200ns_1000ns.png differ diff --git a/benchmarks/lockhammer/scripts/lh_sweeptest_cfg.yaml b/benchmarks/lockhammer/scripts/lh_sweeptest_cfg.yaml index 5bb0e84..18ecee7 100644 --- a/benchmarks/lockhammer/scripts/lh_sweeptest_cfg.yaml +++ b/benchmarks/lockhammer/scripts/lh_sweeptest_cfg.yaml @@ -52,7 +52,7 @@ globalcfg: ## sweeptest: enabled: True - safemode: False + safemode: True cmd: - lh_cas_event_mutex - lh_cas_lockref diff --git a/benchmarks/lockhammer/scripts/lockhammer-all.csv.xz b/benchmarks/lockhammer/scripts/lockhammer-all.csv.xz new file mode 100644 index 0000000..e61864d Binary files /dev/null and b/benchmarks/lockhammer/scripts/lockhammer-all.csv.xz differ diff --git a/benchmarks/lockhammer/scripts/lockhammer-jupyter-notebook.ipynb b/benchmarks/lockhammer/scripts/lockhammer-jupyter-notebook.ipynb new file mode 100644 index 0000000..3b0c9c6 --- /dev/null +++ b/benchmarks/lockhammer/scripts/lockhammer-jupyter-notebook.ipynb @@ -0,0 +1,232 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# License" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) 2018, ARM Limited. All rights reserved.\n", + "\n", + "SPDX-License-Identifier: BSD-3-Clause\n", + "\n", + "Redistribution and use in source and binary forms, with or without\n", + "modification, are permitted provided that the following conditions are met:\n", + "\n", + "Redistributions of source code must retain the above copyright notice, this\n", + "list of conditions and the following disclaimer.\n", + "\n", + "Redistributions in binary form must reproduce the above copyright notice, this\n", + "list of conditions and the following disclaimer in the documentation and/or\n", + "other materials provided with the distribution.\n", + "\n", + "Neither the name of ARM Limited nor the names of its contributors may be used\n", + "to endorse or promote products derived from this software without specific\n", + "prior written permission.\n", + "\n", + "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n", + "AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n", + "IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n", + "DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\n", + "FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n", + "DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n", + "SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n", + "CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR\n", + "TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n", + "OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prerequisite Libraries" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- ## python3\n", + "`apt install python3 python3-pip`\n", + "- ## jupyter-notebook\n", + "`apt install jupyter-notebook`\n", + "- ## matplotlib seaborn pandas numpy\n", + "`pip3 install matplotlib seaborn pandas numpy`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Matplotlib and Seaborn Settings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "import pandas as pd\n", + "import numpy as np\n", + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "sns.set()\n", + "\n", + "# default 12 colors and markers\n", + "default_palette = [\n", + " '#765f97', #Purple\n", + " '#1b9e77', #Dark Green\n", + " '#8c5c20', #Brown\n", + " '#0038bd', #Blue\n", + " '#cf364a', #Red\n", + " '#343434', #Jet Black\n", + " '#878681', #Titanium Gray\n", + " '#f561dd', #Magenta\n", + " '#a6cee3', #Calico Blue\n", + " '#dea0dd', #Plum\n", + " '#7fc97f', #Grass Green\n", + " '#fdc086', #Pale Yellow\n", + " ]\n", + "\n", + "default_markers=['^', '*', 'd', 'x', 'D', 'o', 'v', 's', 'p', '>', '<', '.']\n", + "default_marker_size = 100\n", + "\n", + "# seaborn settings\n", + "sns.set(context=\"notebook\", style=\"darkgrid\", font_scale=2, rc={\"lines.linewidth\": 3, \"xtick.major.size\": 4, \"ytick.major.size\": 4})\n", + "sns.set_palette(default_palette)\n", + "sns.palplot(sns.color_palette())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Lockhammer Common Settings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# common variables\n", + "lock_workloads = [\"cas_event_mutex\", \"cas_lockref\", \"cas_rw_lock\", \"incdec_refcount\", \"osq_lock\", \"queued_spinlock\", \"ticket_spinlock\", \"jvm_objectmonitor\", \"swap_mutex\", \"tbb_spin_rw_mutex\", \"event_mutex\", \"empty\"]\n", + "lock_hosts = [\"x86-1\", \"x86-2\", \"x86-3\", \"x86-4\"]\n", + "lock_parameters = [[\"-c\", \"200ns\", \"-p\", \"1000ns\"]]\n", + "exectx_count_max = 88\n", + "exectx_count_gap = 4\n", + "lh_csv_result_header = [\"num_threads\", \"avg_exectx\", \"scheduled_time_per_access\", \"real_time_per_access\", \"access_rate\", \"avg_lock_depth\",\n", + " \"date\", \"fqdn\", \"exec\", \"t\", \"tv\", \"a\", \"av\", \"c\", \"cv\", \"p\", \"pv\", \"o\", \"ov\"]\n", + "\n", + "param_name = \"contended_system_latency (ns)\"\n", + "lock_yaxis_name = param_name\n", + "\n", + "# lockhammer-all.csv.xz is a xz-compressed aggregated file of different machines' raw csv result\n", + "raw_csv_filename = \"lockhammer-all.csv.xz\"\n", + "raw_df = pd.read_csv(raw_csv_filename, sep=', ', header=None, names=lh_csv_result_header, engine='python')\n", + "\n", + "# common functions\n", + "def plot_lines_only(dataf):\n", + " # each test repeat 9 times, but we only plot the median latency (access_rate)\n", + " median_list = []\n", + " for hst, grp0 in dataf.groupby(\"host\"):\n", + " for nth, grp1 in grp0.groupby(\"num_threads\"):\n", + " median_list.append({\"host\": hst, \"num_threads\": nth, param_name: grp1.median()[param_name]})\n", + " median_df = pd.DataFrame(median_list)\n", + " \n", + " from matplotlib.colors import ListedColormap\n", + " cmap = ListedColormap(sns.color_palette(default_palette).as_hex())\n", + " for i, (hst, grp2) in enumerate(median_df.groupby(\"host\")):\n", + " plt.plot(\"num_threads\", param_name, data=grp2, color=cmap(i))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Lockhammer all workloads, raw contended system latency, 2018.11.06." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "scrolled": false + }, + "outputs": [], + "source": [ + "# use lmplot (not catplot) to plot raw system latencies\n", + "for param in lock_parameters:\n", + " for workload in lock_workloads:\n", + " tidy_df = pd.DataFrame()\n", + " for sut in sorted(lock_hosts):\n", + " host_df = raw_df.loc[(raw_df['fqdn'].str.startswith(sut) & raw_df['exec'].str.endswith(workload))]\n", + " test_df = host_df.loc[(host_df['cv'] == param[1]) & (host_df['pv'] == param[3])]\n", + " copy_df = test_df.copy()\n", + " copy_df['host'] = sut\n", + " all_df = pd.melt(copy_df, id_vars=['host', 'num_threads'], value_vars=['access_rate'], value_name=param_name)\n", + " tidy_df = pd.concat([tidy_df, all_df])\n", + " \n", + " # because lmplot doesn't plot lines, we have to use plot_lines_only to plot them\n", + " sns.lmplot(x=\"num_threads\", y=param_name, hue=\"host\", data=tidy_df, x_estimator=np.median, x_ci=50,\n", + " height=10, aspect=2, fit_reg=False, markers=default_markers[:len(lock_hosts)], scatter_kws={\"s\": default_marker_size})\n", + " \n", + " # plot lines which connect lmplot dots\n", + " plot_lines_only(tidy_df)\n", + " \n", + " # change title / axis and save the figure\n", + " plt.title(\"lockhammer workload: {}, critical_time: {}, parallel_time: {}\".format(workload, param[1], param[3]))\n", + " plt.xlim(0, exectx_count_max)\n", + " plt.xticks(np.arange(0, exectx_count_max+1, exectx_count_gap))\n", + " plt.savefig(\"github_lockhammer_all_common_20181106_{}_{}_{}.png\".format(workload, param[1], param[3]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/lockhammer/scripts/test_lockhammer.py b/benchmarks/lockhammer/scripts/test_lockhammer.py index 84922e9..103ce90 100755 --- a/benchmarks/lockhammer/scripts/test_lockhammer.py +++ b/benchmarks/lockhammer/scripts/test_lockhammer.py @@ -320,7 +320,7 @@ def generate_sweeptest(className, lhCfg): execDir = default_value(globalCfg, 'execdir', os.path.join("..", "build")) logFile = default_value(globalCfg, 'logfile', None) - safeMode = default_value(sweepCfg, 'safemode', False) + safeMode = default_value(sweepCfg, 'safemode', True) repeatCnt = default_value(sweepCfg, 'repeat', 7) sweepArgu = default_value(sweepCfg, 'sweepargu', 't') arguMax = default_value(sweepCfg, 'argumax', 0)