Skip to content
Permalink
Browse files

Add bitswap test suite (#345)

  • Loading branch information
dirkmc committed Feb 13, 2020
1 parent d077156 commit 212d870cd2f2417a30d59b08413d0e9526da8f7d
@@ -34,10 +34,14 @@ name = "transfer"
instances = { min = 2, max = 64, default = 2 }

[testcases.params]
run_count = { type = "int", desc = "number of iterations of the test", unit = "iteration", default = 1 }
run_timeout_secs = { type = "int", desc = "timeout for an individual run", unit = "seconds", default = 90 }
leech_count = { type = "int", desc = "number of leech nodes", unit = "peers", default = 1 }
passive_count = { type = "int", desc = "number of passive nodes (neither leech nor seed)", unit = "peers", default = 0 }
timeout_secs = { type = "int", desc = "timeout", unit = "seconds", default = 60 }
timeout_secs = { type = "int", desc = "timeout", unit = "seconds", default = 300 }
bstore_delay_ms = { type = "int", desc = "blockstore get / put delay", unit = "milliseconds", default = 5 }
request_stagger = { type = "int", desc = "time between each leech's first request", unit = "ms", default = 0}
file_size = { type = "int", desc = "file size", unit = "bytes", default = 4194304 }
latency_ms = { type = "int", desc = "latency", unit = "ms", default = 5 }
bandwidth_mb = { type = "int", desc = "bandwidth", unit = "Mib", default = 1024 }
parallel_gen_mb = { type = "int", desc = "maximum allowed size of seed data to generate in parallel", unit = "Mib", default = 100 }
@@ -0,0 +1,109 @@
'use strict'

function parseMetrics (data) {
const metrics = []
for (const line of data.toString().split('\n')) {
let log = {}
try {
log = JSON.parse(line)
} catch (e) {
}

if ((log.event || {}).type === 'metric') {
const metric = log.event.metric
const parts = metric.name.split(/\//)
// [ 'latencyMS:100', 'bandwidthMB:1024', 'run:1', 'seq:2', 'file-size:10485760', 'Seed:1', 'msgs_rcvd' ]
if (parts.length !== 7) {
throw new Error(`Unexpected metric format ${metric}`)
}

const [latencyDef, bandwidthDef, runDef, seqDef, fileSizeDef, nodeTypeDef, metricName] = parts
const latencyMS = latencyDef.split(':')[1]
const bandwidthMB = bandwidthDef.split(':')[1]
const run = runDef.split(':')[1]
const seq = seqDef.split(':')[1]
const fileSize = fileSizeDef.split(':')[1]
const [nodeType, nodeTypeIndex] = nodeTypeDef.split(':')

metrics.push({ run, seq, latencyMS, bandwidthMB, fileSize, nodeType, nodeTypeIndex, name: metricName, value: metric.value })
}
}

return metrics
}

function groupBy (arr, key) {
const res = {}
for (const i of arr) {
const val = i[key]
res[val] = res[val] || []
res[val].push(i)
}
return res
}

function parseArgs (required, defaults) {
const res = {}
const args = process.argv.slice(2)
for (let i = 0; i < args.length; i++) {
const arg = args[i]
if (arg[0] == '-') {
let k, v
if (arg[1] == '-') {
[k, v] = arg.substring(2).split('=')
} else {
k = arg.substring(1)
v = args[i + 1]
}
if (!k || v == null) {
throw new Error(usage())
}
res[k] = v
}
}

for (const [k, v] of Object.entries(defaults)) {
if (res[k] == null) {
res[k] = v
}
}

for (const req of required) {
if (res[req] == null) {
throw new Error(usage())
}
}

return res
}

const lineColors = [
['#bbe1fa', '#3282b8', '#0f4c75', '#1b262c'],
['#f1bc31', '#e25822', '#b22222', '#7c0a02'],
['#64e291', '#a0cc78', '#589167', '#207561'],
]
const usedColors = []
function getLineColor (branch, seeds, leeches) {
const branchIndex = getBranchIndex(branch, seeds, leeches)
const branchColors = lineColors[branchIndex % lineColors.length]
const colorIndex = usedColors[branchIndex].count % branchColors.length
usedColors[branchIndex].count++
return branchColors[colorIndex]
}

function getBranchIndex (branch, seeds, leeches) {
for (const [i, b] of Object.entries(usedColors)) {
if (b.name === branch) {
return i
}
}
usedColors.push({ name: branch, count: 0 })
return usedColors.length - 1
}

module.exports = {
parseMetrics,
groupBy,
parseArgs,
getLineColor
}
@@ -0,0 +1,43 @@
'use strict'

const { parseMetrics, groupBy } = require('../common')
const fs = require('fs')

const [,, basePath] = process.argv
if (!basePath) {
throw new Error("usage: node peer-choice-process.js <basepath>")
}

run()

function run () {
const data = fs.readFileSync(0).toString() // STDIN_FILENO = 0
const metrics = parseMetrics(data)

const byNodeType = groupBy(metrics, 'nodeType')
for (const [nodeType, metrics] of Object.entries(byNodeType)) {
const byLatency = groupBy(metrics, 'latencyMS')
for (const [latencyMS, metrics] of Object.entries(byLatency)) {
const byName = groupBy(metrics, 'name')
for (const [name, metrics] of Object.entries(byName)) {
const byRun = groupBy(metrics, 'run')
const points = []
for (const runNum of Object.keys(byRun).sort(i => parseInt(i))) {
const metrics = byRun[runNum]
const byFileSize = groupBy(metrics, 'fileSize')
for (const [fileSize, metrics] of Object.entries(byFileSize)) {
const average = metrics.reduce((a, m) => a + m.value, 0) / metrics.length
points.push([fileSize, average])
}
}
writeResults(nodeType, latencyMS, name, 'average', points)
}
}
}
}

function writeResults (nodeType, latencyMS, name, statName, points) {
const filePath = basePath + `.${nodeType}.${latencyMS}.${name}.${statName}.csv`
const content = points.map(p => p.join(',')).join('\n') + '\n'
fs.writeFileSync(filePath, content)
}
@@ -0,0 +1,127 @@
'use strict'

// node chart.js \
// -d ./results/bw1024MB-3x3 \
// -branch master \
// -m blks_sent \
// -b 1024 \
// -xlabel 'File size (MB)' \
// -ylabel 'Blocks Sent' \
// -xscale '9.53674316e-7' \
// && gnuplot ./results/bw1024MB-3x3/master.blks_sent.plot > ./results/bw1024MB-3x3/bw1024MB-3x3.svg

const { parseArgs, getLineColor } = require('../common')
const fs = require('fs')
const path = require('path')

const args = parseArgs(['d', 'm'], {
'b': 1024, // bandwidth in MB
'xscale': 1,
'yscale': 1
})

function getPlotfile () {
let plotFile = `
# Output W3C Scalable Vector Graphics
set terminal svg
# Read comma-delimited data from file
set datafile separator comma
# Put the line labels at the top left
set key left
`

if (args.xlabel) {
plotFile += `set xlabel '${args.xlabel}'\n`
}

if (args.ylabel) {
plotFile += `set ylabel '${args.ylabel}'\n`
}

return plotFile
}

function run (metricName, bandwidthMB, branch) {
let outFiles = parseOutputFiles()
outFiles = outFiles.filter((file) => file.name === metricName)
if (branch) {
outFiles = outFiles.filter((file) => file.branch === branch)
}
outputPlot (metricName, outFiles, bandwidthMB, branch)
}

// Files are in a directory with subdirs for each branch, eg
// results/master/...
// results/mybranch/...
function parseOutputFiles () {
const res = []
const dir = args.d
const subdirs = fs.readdirSync(dir)
for (const subdir of subdirs) {
const subdirPath = path.join(dir, subdir)
if (fs.lstatSync(subdirPath).isDirectory()) {
const files = fs.readdirSync(subdirPath)
for (const file of files) {
const matches = file.match(/(([0-9])+sx([0-9])+l)\.Seed\.([0-9]+)\.(.+)\.average\.csv/)
if (matches) {
const [, label, seeds, leeches, latencyMS, name] = matches
const filePath = path.join(subdirPath, file)

try {
fs.accessSync(filePath)
res.push({ branch: subdir, name, seeds, leeches, latencyMS, filePath })
} catch (e) {
}
}
}
}
}
return res
}

function outputPlot (metricName, outFiles, bandwidthMB, branch) {
const scaledCols = `(column(1)*(${args.xscale})):(column(2)*(${args.yscale}))`

let output = getPlotfile()
output += `set title 'Bitswap (${bandwidthMB}MB bandwidth)'\n`

const plotArgs = []
for (const [i, { branch, seeds, leeches, latencyMS, filePath }] of outFiles.entries()) {
const id = `${branch}${seeds}x${leeches}x${latencyMS}`

const showLine = true //branch === 'master' && seeds > 1
const fn = `f${id}(x)`

if (showLine) {
output += `${fn} = a${id}*x + b${id}\n`
output += `fit ${fn} '${filePath}' using ${scaledCols} via a${id},b${id}\n`
// output += `${fn} = a${id}*x**2 + b${id}*x + c${id}\n`
// output += `fit ${fn} '${filePath}' using 1:2 via a${id},b${id},c${id}\n`
}

const lineColor = getLineColor(branch)
const title = `${branch}: ${latencyMS}ms latency`
plotArgs.push(`'${filePath}' using ${scaledCols} with points title "" lc rgb '${lineColor}' pointtype 3 pointsize 0.5`)
// plotArgs.push(`'${filePath}' smooth csplines title "${title}" lc rgb '${lineColor}' linewidth 2`)
if (showLine) {
plotArgs.push(`${fn} title "${title}" lc rgb '${lineColor}' linewidth 4`)
}
}
const plotCmd = 'plot ' + plotArgs.join(',\\\n ')
output += plotCmd + '\n'

let fileName = `${metricName}.plot`
if (branch) {
fileName = `${branch}.${fileName}`
}
const filePath = path.join(args.d, fileName)
fs.writeFileSync(filePath, output)
}

function usage () {
console.log("usage: node graph.js -d <directory> -m <metric> -branch <branch> -b <bandwidthMB>")
}

run(args.m, args.b, args.branch)
@@ -0,0 +1,71 @@
#!/bin/bash -x

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

OUTPUT_DIR=$1

if [ -z "$OUTPUT_DIR" ]; then
TIMESTAMP=$(date +%Y-%m-%d-%T)
OUTPUT_DIR="/tmp/bitswap-tuning-output/${TIMESTAMP}"
fi

mkdir -p $OUTPUT_DIR

runTest () {
REF=$1
REF_NAME=$2
LABEL=$3
RUN_COUNT=$4
SEEDS=$5
LEECHES=$6
((INSTANCES=$SEEDS+$LEECHES))
LATENCY_MS=$7
BANDWIDTH_MB=$8
FILESIZE=$9
SEED_LATENCY=${10}
TIMEOUT_SECS=${11}

BRANCHES_DIR="${OUTPUT_DIR}/branches"
BRANCH_DIR="${BRANCHES_DIR}/${REF_NAME}"
mkdir -p $BRANCH_DIR
OUTFILE_BASE="${BRANCH_DIR}/${SEEDS}sx${LEECHES}l-${LATENCY_MS}ms-bw${BANDWIDTH_MB}.${LABEL}"
OUTFILE_RAW="${OUTFILE_BASE}.raw"
OUTFILE_CSV_BASE="${BRANCH_DIR}/${SEEDS}sx${LEECHES}l"
./testground run single bitswap-tuning/transfer \
--builder=docker:go \
--runner=local:docker \
--build-cfg bypass_cache=true \
--dep="github.com/ipfs/go-bitswap=$REF" \
--instances=$INSTANCES \
--test-param timeout_secs=$TIMEOUT_SECS \
--test-param run_count=$RUN_COUNT \
--test-param leech_count=$LEECHES \
--test-param latency_ms=$LATENCY_MS \
--test-param bandwidth_mb=$BANDWIDTH_MB \
--test-param file_size=$FILESIZE \
--test-param seed_latency_ms=$SEED_LATENCY \
--run-cfg log_file=$OUTFILE_RAW
RUN_ID=`ls -lt ~/.testground/local_docker/outputs/bitswap-tuning | head -2 | tail -1 | awk '{print $NF}'`
OUTZIP="${OUTPUT_DIR}/${RUN_ID}.zip"
./testground collect --runner=local:docker --output=$OUTZIP $RUN_ID
unzip $OUTZIP -d $OUTPUT_DIR
cat ${OUTPUT_DIR}/${RUN_ID}/single/*/run.out | node $SCRIPT_DIR/aggregate.js $OUTFILE_CSV_BASE
node $SCRIPT_DIR/chart.js -d $BRANCHES_DIR -m blks_sent -b 64 -xlabel 'File size (MB)' -ylabel 'Blocks Sent' -xscale '9.53674316e-7' -branch $REF_NAME
gnuplot $BRANCHES_DIR/${REF_NAME}.blks_sent.plot > $BRANCHES_DIR/${REF_NAME}.blks_sent.svg
}

BW=64 # bandwidth in MB
LTCY=100
SIZES=1048576,2097152,4194304,8388608,16777216,33554432,67108864
SEED_LTCY=100,500,1000
TIMEOUT=1200
ITERATIONS=5
LABEL='1-64MB'

# 4 seed / 1 leech
runTest 'dcfe40e' 'old' $LABEL $ITERATIONS 4 1 $LTCY $BW $SIZES $SEED_LTCY $TIMEOUT

# 4 seed / 1 leech
runTest '65321e4' 'new' $LABEL $ITERATIONS 4 1 $LTCY $BW $SIZES $SEED_LTCY $TIMEOUT

echo "Output: $OUTPUT_DIR"

0 comments on commit 212d870

Please sign in to comment.
You can’t perform that action at this time.