diff --git a/src/db-data-processing.ts b/src/db-data-processing.ts index 71347ff2..1e55e231 100644 --- a/src/db-data-processing.ts +++ b/src/db-data-processing.ts @@ -180,6 +180,7 @@ function findMeasurements( if ( mm.envId == row.envid && mm.commitId == row.commitid && + mm.runId == row.runid && mm.criterion.name == row.criterion ) { m = mm; diff --git a/src/stats-data-prep.ts b/src/stats-data-prep.ts index ee5bdcb6..7ef832b3 100644 --- a/src/stats-data-prep.ts +++ b/src/stats-data-prep.ts @@ -177,6 +177,7 @@ function assertBasicPropertiesOfSortedMeasurements( function addOrGetCompareStatsRow( variants: Map, + countsAndMissing: VariantCountAndMissing | null, measurements: Measurements, bench: ProcessedResult ): CompareStatsRow { @@ -196,17 +197,17 @@ function addOrGetCompareStatsRow( let row = variants.get(key); if (row === undefined) { row = { - benchId: { b, e, s, v, c, ea }, + benchId: { b, e, s, v, c, i, ea }, details: { cmdline: measurements.runSettings.simplifiedCmdline, envId, hasProfiles: false, hasWarmup: false, - numV: 0, - numC: 0, - numI: 0, - numEa: 0, - numEnv: 0 + numV: countsAndMissing === null ? 0 : countsAndMissing.numV, + numC: countsAndMissing === null ? 0 : countsAndMissing.numC, + numI: countsAndMissing === null ? 0 : countsAndMissing.numI, + numEa: countsAndMissing === null ? 0 : countsAndMissing.numEa, + numEnv: countsAndMissing === null ? 0 : countsAndMissing.numEnv } }; variants.set(key, row); @@ -221,7 +222,7 @@ function addMissingCompareStatsRow( base: string, change: string ): void { - const row = addOrGetCompareStatsRow(variants, measurements, bench); + const row = addOrGetCompareStatsRow(variants, null, measurements, bench); if (!row.missing) { row.missing = []; @@ -263,7 +264,7 @@ export function countVariantsAndDropMissing( let numC = 0; let numI = 0; let numEa = 0; - let numEnv = 1; + let numEnv = 0; function dropAsMissing(i: number): void { addMissingCompareStatsRow(missing, measurements[i], bench, base, change); @@ -290,37 +291,29 @@ export function countVariantsAndDropMissing( const base = measurements[i]; - if (lastEnvId === -1) { + if (lastEnvId !== base.envId) { lastEnvId = base.envId; - lastVarValue = base.runSettings.varValue; - lastCores = base.runSettings.cores; - lastInputSize = base.runSettings.inputSize; - lastExtraArgs = base.runSettings.extraArgs; - } else { - if (lastEnvId !== base.envId) { - lastEnvId = base.envId; - numEnv += 1; - } + numEnv += 1; + } - if (lastVarValue !== base.runSettings.varValue) { - lastVarValue = base.runSettings.varValue; - numV += 1; - } + if (lastVarValue !== base.runSettings.varValue) { + lastVarValue = base.runSettings.varValue; + numV += 1; + } - if (lastCores !== base.runSettings.cores) { - lastCores = base.runSettings.cores; - numC += 1; - } + if (lastCores !== base.runSettings.cores) { + lastCores = base.runSettings.cores; + numC += 1; + } - if (lastInputSize !== base.runSettings.inputSize) { - lastInputSize = base.runSettings.inputSize; - numI += 1; - } + if (lastInputSize !== base.runSettings.inputSize) { + lastInputSize = base.runSettings.inputSize; + numI += 1; + } - if (lastExtraArgs !== base.runSettings.extraArgs) { - lastExtraArgs = base.runSettings.extraArgs; - numEa += 1; - } + if (lastExtraArgs !== base.runSettings.extraArgs) { + lastExtraArgs = base.runSettings.extraArgs; + numEa += 1; } if (isLastItem) { @@ -387,6 +380,7 @@ export async function calculateChangeStatsForBenchmark( // - calculate the change statistics for each criterion const counts = await computeStatisticsAndInlinePlot( variants, + countsAndMissing, bench, measurements, baseOffset, @@ -443,6 +437,7 @@ export function getMsFlattenedAndSorted( async function computeStatisticsAndInlinePlot( variants: Map, + countsAndMissing: VariantCountAndMissing, bench: ProcessedResult, measurements: Measurements[], baseOffset: number, @@ -462,7 +457,12 @@ async function computeStatisticsAndInlinePlot( const { sortedBase, sortedChange } = getMsFlattenedAndSorted(base, change); const changeStats = calculateChangeStatistics(sortedBase, sortedChange); - const row = addOrGetCompareStatsRow(variants, change, bench); + const row = addOrGetCompareStatsRow( + variants, + countsAndMissing, + change, + bench + ); // add the various details row.details.hasWarmup = siteConfig.canShowWarmup(change.values); diff --git a/src/views/compare/stats-row.html b/src/views/compare/stats-row.html index 207712ec..966acd3e 100644 --- a/src/views/compare/stats-row.html +++ b/src/views/compare/stats-row.html @@ -3,10 +3,10 @@ let args = ''; -if (stats.details.numV > 0) { args += stats.benchId.v + ' '; } -if (stats.details.numC > 0) { args += stats.benchId.c + ' '; } -if (stats.details.numI > 0) { args += stats.benchId.i + ' '; } -if (stats.details.numEa > 0) { args += stats.benchId.ea + ' '; } +if (stats.details.numV > 1) { args += stats.benchId.v + ' '; } +if (stats.details.numC > 1) { args += stats.benchId.c + ' '; } +if (stats.details.numI > 1) { args += stats.benchId.i + ' '; } +if (stats.details.numEa > 1) { args += stats.benchId.ea + ' '; } if (args.length > 0) { args = `${args.trim()}`; @@ -14,7 +14,7 @@ if (stats.missing) { %} -%}{%= stats.benchId.b %}{%= args %} +%}{%= stats.benchId.b %}{%- args %} Missing data for {% const byCommitId = new Map(); for (const m of stats.missing) { @@ -46,7 +46,7 @@ if (hasTotal) { %} -{%= stats.benchId.b %}{%= args %} +{%= stats.benchId.b %}{%- args %} {% if (stats.exeStats) { %}{%- include('compare/stats-row-across-exes.html', { diff --git a/tests/db-data-processing.test.ts b/tests/db-data-processing.test.ts index cb06d1c9..de522e48 100644 --- a/tests/db-data-processing.test.ts +++ b/tests/db-data-processing.test.ts @@ -7,7 +7,7 @@ import { ResultsByExeSuiteBenchmark, ResultsBySuiteBenchmark } from '../src/stats-data-prep.js'; -import { ProcessedResult } from '../src/db.js'; +import { MeasurementData, ProcessedResult } from '../src/db.js'; const dataJsSOM = JSON.parse( readFileSync( @@ -175,4 +175,75 @@ describe('collateMeasurements()', () => { expect(m2.values[0]).toHaveLength(120); }); }); + + describe('needs to distinguish different run ids', () => { + function createMeasure( + runid: number, + inputsize: string, + commitid: string + ): MeasurementData { + return { + expid: 1, + runid, + trialid: 1, + commitid, + + bench: 'b', + exe: 'e', + suite: 's', + + cmdline: 'b e s ' + inputsize, + varvalue: null, + cores: null, + inputsize, + extraargs: null, + + iteration: 1, + invocation: 1, + warmup: null, + + criterion: 'total', + unit: 'ms', + value: 1, + + envid: 1 + }; + } + + const data: MeasurementData[] = [ + createMeasure(1, '1', 'a'), + createMeasure(1, '1', 'b'), + createMeasure(2, '2', 'a'), + createMeasure(2, '2', 'b') + ]; + + const result: ResultsByExeSuiteBenchmark = collateMeasurements(data); + + it('should have 1 executor', () => { + expect(result.size).toBe(1); + expect([...result.keys()]).toEqual(['e']); + }); + + it('should have 1 suite', () => { + const suites = result.get('e'); + expect(suites.size).toBe(1); + expect([...suites.keys()]).toEqual(['s']); + }); + + it('should have the expected benchmark', () => { + const suites = result.get('e'); + const bs = suites.get('s'); + expect(bs.benchmarks.size).toBe(1); + + expect([...bs.benchmarks.keys()]).toEqual(['b']); + }); + + it('should have 4 measurements', () => { + const suites = result.get('e'); + const bs = suites.get('s'); + const b = bs.benchmarks.get('b'); + + expect(b.measurements).toHaveLength(4); + }); + }); }); diff --git a/tests/stats-data-prep.test.ts b/tests/stats-data-prep.test.ts index dac8fba8..90777068 100644 --- a/tests/stats-data-prep.test.ts +++ b/tests/stats-data-prep.test.ts @@ -358,6 +358,16 @@ describe('calculateChangeStatsForBenchmark()', () => { it('should not have dropped any data', () => { expect(nbody.measurements).toHaveLength(2); expect(stats.stats).toHaveLength(1); + + for (const propName of ['b', 'e', 's', 'v', 'c', 'i', 'ea']) { + expect(stats.stats[0].benchId).toHaveProperty(propName); + } + + expect(stats.stats[0].details.numV).toBe(0); + expect(stats.stats[0].details.numC).toBe(1); + expect(stats.stats[0].details.numI).toBe(0); + expect(stats.stats[0].details.numEa).toBe(1); + expect(stats.stats[0].details.numEnv).toBe(1); }); it('should have added the expected statistics', () => {