From d8297e00ef26f98cf5b03a098afad963a2c87ff8 Mon Sep 17 00:00:00 2001 From: Nikhil Thorat Date: Thu, 19 Dec 2019 15:49:29 -0500 Subject: [PATCH 1/2] [NOT READY] Update benchmarks to show aggregate kernel timing or individual kernel timing. --- tfjs-core/benchmarks/index.html | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tfjs-core/benchmarks/index.html b/tfjs-core/benchmarks/index.html index 5211d63b771..fc4678075e5 100644 --- a/tfjs-core/benchmarks/index.html +++ b/tfjs-core/benchmarks/index.html @@ -72,14 +72,7 @@

TensorFlow.js Model Benchmark

- - - - - - - - +
KernelTime(ms)InputsOutputGPUPrograms
@@ -105,11 +98,14 @@

TensorFlow.js Model Benchmark

runBenchmark(); }, backend: 'wasm', + kernelTiming: 'aggregate', }; const modalDiv = document.getElementById('modal-msg'); const timeTable = document.querySelector('#timings tbody'); const envDiv = document.getElementById('env'); + const kernelsTableHead = document.getElementById('kernels-thead'); + let model, predict, chartWidth; async function showMsg(message) { @@ -137,6 +133,18 @@

TensorFlow.js Model Benchmark

} `; } + async function showTableHeaders() { + kernelsTableHead.innerText = ''; + await tf.nextFrame(); + const rows = ['Kernel', 'Time(ms)']; + if (state.kernelTiming === 'individual') { + rows.push('Inputs', 'Output', 'GPUPrograms'); + } + appendRow(kernelsTableHead, ...rows); + + await tf.nextFrame(); + } + function appendRow(tbody, ...cells) { const tr = document.createElement('tr'); cells.forEach(c => { @@ -330,6 +338,7 @@

TensorFlow.js Model Benchmark

async function runBenchmark() { const benchmark = benchmarks[state.benchmark]; + await showTableHeaders(); await loadAndRecordTime(benchmark); await warmUpAndRecordTime(); await showMsg('Waiting for GC'); @@ -356,6 +365,7 @@

TensorFlow.js Model Benchmark

gui.add(state, 'backend', ['wasm', 'webgl', 'cpu']).onChange(backend => { tf.setBackend(backend); }); + gui.add(state, 'kernelTiming', ['aggregate', 'individual']); gui.add(state, 'run'); showVersions(); From 3f85bec5ff0ec78b5952d3c806bfa40d491c6ef4 Mon Sep 17 00:00:00 2001 From: Nikhil Thorat Date: Fri, 20 Dec 2019 12:04:21 -0500 Subject: [PATCH 2/2] save --- tfjs-core/benchmarks/index.html | 86 ++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/tfjs-core/benchmarks/index.html b/tfjs-core/benchmarks/index.html index fc4678075e5..73afdccff83 100644 --- a/tfjs-core/benchmarks/index.html +++ b/tfjs-core/benchmarks/index.html @@ -50,26 +50,6 @@

TensorFlow.js Model Benchmark

-
-
Number of tensors
-
-
-
- - - -
-
-
-
Number of bytes used
-
-
-
- - - -
-
@@ -105,6 +85,7 @@

TensorFlow.js Model Benchmark

const timeTable = document.querySelector('#timings tbody'); const envDiv = document.getElementById('env'); const kernelsTableHead = document.getElementById('kernels-thead'); + const kernelTable = document.querySelector('#kernels tbody'); let model, predict, chartWidth; @@ -133,12 +114,16 @@

TensorFlow.js Model Benchmark

} `; } - async function showTableHeaders() { + async function setupTable() { kernelsTableHead.innerText = ''; + kernelTable.innerHTML = ''; await tf.nextFrame(); - const rows = ['Kernel', 'Time(ms)']; + const rows = ['Kernel', 'Time(ms)']; if (state.kernelTiming === 'individual') { - rows.push('Inputs', 'Output', 'GPUPrograms'); + rows.push('Inputs', 'Output'); + if (state.backend === 'webgl') { + rows.push('GPUPrograms'); + } } appendRow(kernelsTableHead, ...rows); @@ -181,7 +166,12 @@

TensorFlow.js Model Benchmark

async function loadAndRecordTime(benchmark) { await showMsg('Loading the model'); const start = performance.now(); - model = await benchmark.load(); + if (benchmark.model == null) { + model = await benchmark.load(); + benchmark.model = model; + } else { + model = benchmark.model; + } predict = benchmark.predictFunc(); const elapsed = performance.now() - start; @@ -217,11 +207,11 @@

TensorFlow.js Model Benchmark

chartWidth = document.querySelector("#perf-trendline-container").getBoundingClientRect().width; const times = []; - const numTensors = []; - const numBytes = []; + const numLeakedTensors = []; for (let i = 0; i < state.numRuns; i++) { const start = performance.now(); + const tensorsBefore = tf.memory().numTensors; let res = predict(model); if (res instanceof Promise) { res = await res; @@ -235,22 +225,19 @@

TensorFlow.js Model Benchmark

times.push(performance.now() - start); const memInfo = tf.memory(); - numTensors.push(memInfo.numTensors); - numBytes.push(memInfo.numBytes); + const leakedTensors = memInfo.numTensors - tensorsBefore; + numLeakedTensors.push(leakedTensors); } const forceInferenceTrendYMinToZero = true; populateTrendline(document.querySelector("#perf-trendline-container"), times, forceInferenceTrendYMinToZero, printTime); - populateTrendline(document.querySelector("#mem-trendline-container"), numTensors); - - const forceBytesTrendlineYMinToZero = false; - populateTrendline(document.querySelector("#bytes-trendline-container"), numBytes, forceBytesTrendlineYMinToZero, d => `${(d / 1e6).toPrecision(3)} MB`); await showMsg(null); const average = times.reduce((acc, curr) => acc + curr, 0) / times.length; const min = Math.min(...times); appendRow(timeTable, `Subsequent average(${state.numRuns} runs)`, printTime(average)); appendRow(timeTable, 'Best time', printTime(min)); + appendRow(timeTable, 'Leaked tensors', numLeakedTensors[0]); } async function profileMemory() { @@ -275,12 +262,33 @@

TensorFlow.js Model Benchmark

function showKernelTime(kernels) { const tbody = document.querySelector('#kernels tbody'); - kernels.forEach(k => { - const nameSpan = document.createElement('span'); - nameSpan.setAttribute('title', k.scopes.slice(0, -1).join(' --> ')); - nameSpan.textContent = k.scopes[k.scopes.length - 1]; - appendRow(tbody, nameSpan, k.time.toFixed(2), k.inputs, k.output, k.gpuProgramsInfo); - }); + if (state.kernelTiming === 'individual') { + kernels.forEach(k => { + const nameSpan = document.createElement('span'); + nameSpan.setAttribute('title', k.scopes.slice(0, -1).join(' --> ')); + nameSpan.textContent = k.scopes[k.scopes.length - 1]; + appendRow(tbody, nameSpan, k.time.toFixed(2), k.inputs, k.output, k.gpuProgramsInfo); + }); + } else { + const kernelTotalTime = {}; + kernels.forEach(k => { + const kernelName = k.scopes[0]; + if (kernelTotalTime[kernelName] == null) { + kernelTotalTime[kernelName] = 0; + } + kernelTotalTime[kernelName] += k.time; + }); + + const result = Object.keys(kernelTotalTime) + .map(k => [k, kernelTotalTime[k]]) + .sort((a, b) => b[1] - a[1]); + result.forEach(r => { + const nameSpan = document.createElement('span'); + nameSpan.setAttribute('title', r[0]); + nameSpan.textContent = r[0]; + appendRow(tbody, nameSpan, r[1].toFixed(2)); + }); + } } async function profileKernelTime() { @@ -338,7 +346,7 @@

TensorFlow.js Model Benchmark

async function runBenchmark() { const benchmark = benchmarks[state.benchmark]; - await showTableHeaders(); + await setupTable(); await loadAndRecordTime(benchmark); await warmUpAndRecordTime(); await showMsg('Waiting for GC');