From 0246a71e78c650316724f910e5acf80e3a662908 Mon Sep 17 00:00:00 2001 From: JiaweiZhuang Date: Fri, 13 Dec 2019 21:06:14 -0500 Subject: [PATCH] Plot OpenMPI collectives with EFA --- notebooks/collective_ompi_efa.ipynb | 466 ++++++++++++++++++++++++++++ 1 file changed, 466 insertions(+) create mode 100644 notebooks/collective_ompi_efa.ipynb diff --git a/notebooks/collective_ompi_efa.ipynb b/notebooks/collective_ompi_efa.ipynb new file mode 100644 index 0000000..7b9f67a --- /dev/null +++ b/notebooks/collective_ompi_efa.ipynb @@ -0,0 +1,466 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import numpy as np\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from osu_util import read_osu_log, read_osu_log_multi, plot_osu\n", + "from collective_algo import bcast_algo_intelmpi, bcast_algo_openmpi3, bcast_algo_openmpi4" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1m\u001b[34mopenmpi3-efa\u001b[m\u001b[m/ \u001b[1m\u001b[34mopenmpi4-efa\u001b[m\u001b[m/\n" + ] + } + ], + "source": [ + "ls osu_log_addon/bcast/N8n288/run3/" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "TOP_DIR = './osu_log_addon/bcast/N8n288/run3/'" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['./osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_default.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_algo0.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_algo1.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_algo2.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_algo3.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_algo4.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_algo5.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi3-efa/bcast_algo6.log']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ompi3_dir = TOP_DIR + 'openmpi3-efa/'\n", + "\n", + "ompi3_file_list = [ompi3_dir + 'bcast_default.log'] + [ompi3_dir + 'bcast_algo{}.log'.format(i) for i in range(0, 7)]\n", + "ompi3_file_list" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
default0. ignore1. basic linear2. chain3. pipeline4. split binary tree5. binary tree6. binomial tree
size
192.6698.55410.47182.88630.55632.51126.8290.84
1697.0799.21409.38182.94641.76151.50144.0597.17
256101.00100.61415.33189.48659.22167.07139.22101.55
4096310.66238.32447.87311.471151.44169.95155.36109.27
65536828.06824.603531.291107.224041.93455.52865.02879.15
10485763811.363891.0720917.6210995.3941535.054957.357770.559794.24
\n", + "
" + ], + "text/plain": [ + " default 0. ignore 1. basic linear 2. chain 3. pipeline \\\n", + "size \n", + "1 92.66 98.55 410.47 182.88 630.55 \n", + "16 97.07 99.21 409.38 182.94 641.76 \n", + "256 101.00 100.61 415.33 189.48 659.22 \n", + "4096 310.66 238.32 447.87 311.47 1151.44 \n", + "65536 828.06 824.60 3531.29 1107.22 4041.93 \n", + "1048576 3811.36 3891.07 20917.62 10995.39 41535.05 \n", + "\n", + " 4. split binary tree 5. binary tree 6. binomial tree \n", + "size \n", + "1 632.51 126.82 90.84 \n", + "16 151.50 144.05 97.17 \n", + "256 167.07 139.22 101.55 \n", + "4096 169.95 155.36 109.27 \n", + "65536 455.52 865.02 879.15 \n", + "1048576 4957.35 7770.55 9794.24 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_ompi3 = read_osu_log_multi(ompi3_file_list, ['default'] + bcast_algo_openmpi3)\n", + "df_ompi3[::4]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['./osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_default.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo0.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo1.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo2.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo3.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo4.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo5.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo6.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo7.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo8.log',\n", + " './osu_log_addon/bcast/N8n288/run3/openmpi4-efa/bcast_algo9.log']" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ompi4_dir = TOP_DIR + 'openmpi4-efa/'\n", + "\n", + "ompi4_file_list = [ompi4_dir + 'bcast_default.log'] + [ompi4_dir + 'bcast_algo{}.log'.format(i) for i in range(0, 10)]\n", + "ompi4_file_list" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
default0. ignore1. basic linear2. chain3. pipeline4. split binary tree5. binary tree6. binomial tree7. knomial tree8. scatter_allgather9. scatter_allgather_ring
size
188.8193.44397.86173.51588.11591.91128.6989.5259.20393.18399.18
1693.9995.27399.34175.05589.25149.44131.6888.5858.87392.86403.33
256101.6199.81404.82177.25608.56155.93134.3597.8961.45399.97408.42
4096258.62242.66430.04249.74899.23184.48154.32109.2368.78487.471518.25
65536906.88906.123412.64663.552406.88431.65861.58876.51350.24744.132013.36
10485763081.173146.6820502.736080.7922321.414875.667257.079806.962425.277719.209673.95
\n", + "
" + ], + "text/plain": [ + " default 0. ignore 1. basic linear 2. chain 3. pipeline \\\n", + "size \n", + "1 88.81 93.44 397.86 173.51 588.11 \n", + "16 93.99 95.27 399.34 175.05 589.25 \n", + "256 101.61 99.81 404.82 177.25 608.56 \n", + "4096 258.62 242.66 430.04 249.74 899.23 \n", + "65536 906.88 906.12 3412.64 663.55 2406.88 \n", + "1048576 3081.17 3146.68 20502.73 6080.79 22321.41 \n", + "\n", + " 4. split binary tree 5. binary tree 6. binomial tree \\\n", + "size \n", + "1 591.91 128.69 89.52 \n", + "16 149.44 131.68 88.58 \n", + "256 155.93 134.35 97.89 \n", + "4096 184.48 154.32 109.23 \n", + "65536 431.65 861.58 876.51 \n", + "1048576 4875.66 7257.07 9806.96 \n", + "\n", + " 7. knomial tree 8. scatter_allgather 9. scatter_allgather_ring \n", + "size \n", + "1 59.20 393.18 399.18 \n", + "16 58.87 392.86 403.33 \n", + "256 61.45 399.97 408.42 \n", + "4096 68.78 487.47 1518.25 \n", + "65536 350.24 744.13 2013.36 \n", + "1048576 2425.27 7719.20 9673.95 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_ompi4 = read_osu_log_multi(ompi4_file_list, ['default'] + bcast_algo_openmpi4)\n", + "df_ompi4[::4]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}