Skip to content

Commit

Permalink
TS-33126 Accept performance regression because we now copy all files
Browse files Browse the repository at this point in the history
  • Loading branch information
@DreierF
DreierF committed Apr 5, 2023
1 parent 1f1ccef commit a8ab48d
Showing 1 changed file with 77 additions and 39 deletions.
116 changes: 77 additions & 39 deletions test/systemtest.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import net from "net";
import net from 'net';
import tempfile from 'tempfile';
import { execSync, spawn } from 'child_process';
import path from 'path';
Expand Down Expand Up @@ -94,21 +94,21 @@ const caseStudies = [
},
excludeOrigins: [`'src/app/heroes/*.*'`, `'node_modules/**/*.*'`, `'webpack/**/*'`],
includeOrigins: [],
maxNormTimeFraction: 8.0
maxNormTimeFraction: 9.0
}
];

/**
* Identify the next available port.
*/
async function identifyNextAvailablePort() {
return new Promise( res => {
return new Promise(res => {
const srv = net.createServer();
srv.listen(0, () => {
const port = srv.address().port;
srv.close(_ => res(port))
srv.close(_ => res(port));
});
})
});
}

/**
Expand Down Expand Up @@ -214,7 +214,18 @@ function identifyExpectedButAbsent(actual, expected) {
function startCollector(coverageFolder, logTargetFile, projectId, collectorPort) {
const collector = spawn(
'node',
[`${COLLECTOR_DIR}/dist/src/main.js`, `--port`, collectorPort, `-f`, `${coverageFolder}`, `-l`, `${logTargetFile}`, `-k`, `-e`, `info`],
[
`${COLLECTOR_DIR}/dist/src/main.js`,
`--port`,
collectorPort,
`-f`,
`${coverageFolder}`,
`-l`,
`${logTargetFile}`,
`-k`,
`-e`,
`info`
],
{
env: {
...process.env,
Expand All @@ -225,11 +236,10 @@ function startCollector(coverageFolder, logTargetFile, projectId, collectorPort)
},
TEAMSCALE_PARTITION: 'mockPartition',
TEAMSCALE_BRANCH: 'mockBranch',
detached: false
detached: false
}
);


collector.stdout.on('data', function (data) {
console.log('collector stdout: ' + data.toString());
});
Expand All @@ -247,13 +257,20 @@ function startCollector(coverageFolder, logTargetFile, projectId, collectorPort)
* Start the Web server process.
* @returns {ChildProcessWithoutNullStreams}
*/
function startWebServer(distributionFolder) {
function startWebServer(distributionFolder) {
const webserver = spawn(
'node',
['node_modules/.bin/ws', '--static.maxage', '0',
'--hostname', 'localhost',
'--port', SERVER_PORT,
'--directory', distributionFolder],
[
'node_modules/.bin/ws',
'--static.maxage',
'0',
'--hostname',
'localhost',
'--port',
SERVER_PORT,
'--directory',
distributionFolder
],
{
detached: false,
killSignal: 'SIGKILL',
Expand All @@ -271,7 +288,7 @@ function startCollector(coverageFolder, logTargetFile, projectId, collectorPort)
console.error('webserver stderr: ' + data.toString());
});

webserver.on('close', (code) => {
webserver.on('close', code => {
console.log(`webserver process exited with code ${code}`);
});

Expand Down Expand Up @@ -343,7 +360,7 @@ function summarizePerformanceMeasures(perfMeasuresByStudy) {
return valueInCaseOfZero;
}
return toCheck;
}
};

const result = {};
result[perfType + 'Base'] = basePerfValue;
Expand All @@ -355,7 +372,9 @@ function summarizePerformanceMeasures(perfMeasuresByStudy) {
result[perfType + 'NoInstNormalized'] = noInstPerfDiffToBase;
result[perfType + 'NormalizedDelta'] = withInstPerfDiffToBase - noInstPerfDiffToBase;
result[perfType + 'NormalizedFraction'] = withInstPerfDiffToBase / noInstPerfDiffToBase;
result[perfType + 'NormalizedFraction' + String(fractionFactor)] = Math.round(withInstPerfDiffToBase / fractionFactor) / ifZeroElse(1, Math.round(noInstPerfDiffToBase / fractionFactor));
result[perfType + 'NormalizedFraction' + String(fractionFactor)] =
Math.round(withInstPerfDiffToBase / fractionFactor) /
ifZeroElse(1, Math.round(noInstPerfDiffToBase / fractionFactor));
result[perfType + 'NormalizedFraction'] = withInstPerfDiffToBase / noInstPerfDiffToBase / fractionFactor;
return result;
};
Expand All @@ -380,25 +399,28 @@ function summarizePerformanceMeasures(perfMeasuresByStudy) {
}

results.push({
'study': study.name,
...computePerfStats(study, 'memory', 'memory_mb_peak', 100, (value) => Math.ceil(value)),
...computePerfStats(study, 'time', 'duration_secs', 1, (value) => Number(value.toPrecision(2)))
study: study.name,
...computePerfStats(study, 'memory', 'memory_mb_peak', 100, value => Math.ceil(value)),
...computePerfStats(study, 'time', 'duration_secs', 1, value => Number(value.toPrecision(2)))
});
}
return results;
}
};

const checkPerformanceMeasures = (runtimeResults) => {
for (let i=0; i<runtimeResults.length; i++) {
const checkPerformanceMeasures = runtimeResults => {
for (let i = 0; i < runtimeResults.length; i++) {
const runtimeResult = runtimeResults[i];
// The baseline study is skipped, that is, a +1 is needed:
const study = caseStudies[i+1];
const study = caseStudies[i + 1];

// We only trigger a violation if the time needed for the study itself is larger than 2s
if ('maxNormTimeFraction' in study && runtimeResult.timeWithInstNormalized > 2) {
const maxValue = study.maxNormTimeFraction;
if (runtimeResult.timeNormalizedFraction > maxValue) {
console.error(`Time overhead added by the instrumentation was too high! ${runtimeResult.timeNormalizedFraction} > ${maxValue}`, study.name);
console.error(
`Time overhead added by the instrumentation was too high! ${runtimeResult.timeNormalizedFraction} > ${maxValue}`,
study.name
);
process.exit(6);
}
}
Expand All @@ -407,12 +429,15 @@ function summarizePerformanceMeasures(perfMeasuresByStudy) {
if ('maxNormMemoryFraction100' in study && runtimeResult.memoryNormalizedFraction100 > 2) {
const maxValue = study.maxNormMemoryFraction100;
if (runtimeResult.memoryNormalizedFraction > maxValue) {
console.error(`Memory overhead added by the instrumentation was too high! ${runtimeResult.memoryNormalizedFraction} > ${maxValue}`, study.name);
console.error(
`Memory overhead added by the instrumentation was too high! ${runtimeResult.memoryNormalizedFraction} > ${maxValue}`,
study.name
);
process.exit(7);
}
}
}
}
};

const runtimeResults = aggregatePerformanceResults();
console.log(runtimeResults);
Expand All @@ -430,13 +455,18 @@ function averagePerformance(samples) {
}

function profileTestingInBrowser(studyName) {
const runTestsOnSubjectInBrowser = (studyName) => {
const runTestsOnSubjectInBrowser = studyName => {
const browserPerformanceFile = tempfile('.json');
console.log('## Running Cypress tests on the subject');
const command = `${path.join(PROFILER_ROOT_DIR, 'test', 'scripts', 'profile_testing.sh')} ${studyName} ${SERVER_PORT} ${browserPerformanceFile}`;
const command = `${path.join(
PROFILER_ROOT_DIR,
'test',
'scripts',
'profile_testing.sh'
)} ${studyName} ${SERVER_PORT} ${browserPerformanceFile}`;
execSync(command, { cwd: PROFILER_ROOT_DIR, stdio: 'inherit' });
return JSON.parse(fs.readFileSync(browserPerformanceFile));
}
};

// We do one run that is just for warm-up. Its measures are not considered.
runTestsOnSubjectInBrowser(studyName);
Expand All @@ -459,19 +489,22 @@ function instrumentStudy(study, collectorPort) {
/**
* Attention: This does wildcard expansion!!
* See https://stackoverflow.com/questions/11717281/wildcards-in-child-process-spawn
*/
const excludeArgument =
study.excludeOrigins.length > 0 ? `--exclude-origin ${study.excludeOrigins.join(' ')}` : '';
const includeArgument =
study.includeOrigins.length > 0 ? `--include-origin ${study.includeOrigins.join(' ')}` : '';
*/
const excludeArgument = study.excludeOrigins.length > 0 ? `--exclude-origin ${study.excludeOrigins.join(' ')}` : '';
const includeArgument = study.includeOrigins.length > 0 ? `--include-origin ${study.includeOrigins.join(' ')}` : '';
console.log('Include/exclude arguments: ', includeArgument, excludeArgument);

const performanceFile = tempfile('.json');
execSync(
`${path.join(PROFILER_ROOT_DIR, 'test', 'scripts', 'profile_instrumentation.sh')} ${fullStudyDistPath} ${collectorPort} ${performanceFile} ${excludeArgument} ${includeArgument}`,
`${path.join(
PROFILER_ROOT_DIR,
'test',
'scripts',
'profile_instrumentation.sh'
)} ${fullStudyDistPath} ${collectorPort} ${performanceFile} ${excludeArgument} ${includeArgument}`,
{ cwd: INSTRUMENTER_DIR, stdio: 'inherit' }
);
return JSON.parse(fs.readFileSync(performanceFile));
);
return JSON.parse(fs.readFileSync(performanceFile));
}

function storePerfResult(perfMeasuresByStudy, studyName, perfKey, perf) {
Expand Down Expand Up @@ -541,7 +574,7 @@ async function startTeamscaleMockServer(study) {
}

function checkTeamscaleServerMockInteractions(mockInstance, study) {
const mockedRequests = mockInstance.requests({method: 'POST'}).length;
const mockedRequests = mockInstance.requests({ method: 'POST' }).length;
if (mockedRequests === 0 && Object.keys(study.expectCoveredLines).length > 0) {
throw new Error('No coverage information was sent to the Teamscale mock server!');
} else {
Expand Down Expand Up @@ -581,7 +614,12 @@ await (async function runSystemTest() {
// Run the tests in the browser on the instrumented version
try {
const runtimePerformance = profileTestingInBrowser(study.name);
storePerfResult(perfStore, study.name, KEY_PERF_TESTING_WITH_INSTRUMENTATION, runtimePerformance);
storePerfResult(
perfStore,
study.name,
KEY_PERF_TESTING_WITH_INSTRUMENTATION,
runtimePerformance
);
} finally {
console.log('## Stopping the collector');
await sleep(1000);
Expand Down

0 comments on commit a8ab48d

Please sign in to comment.