From 32ba29a14373d0c89b5f42a095c3448e84d89043 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Sat, 6 Feb 2021 21:46:08 -0700 Subject: [PATCH] devops: introduce compressed dashboard Compressed dashboard is 10 times smaller yet has all the data to render flakiness. Drive-by: remove old dashboard implementations since they are no longer used. --- .../processing/compress_raw_data.js | 34 ++++ .../processing/dashboard_compressed_v1.js | 119 ++++++++++++++ .../processing/dashboard_raw.js | 4 + .../processing/dashboard_v1.js | 102 ------------ .../processing/dashboard_v2.js | 155 ------------------ utils/flakiness-dashboard/processing/index.js | 10 +- utils/flakiness-dashboard/processing/utils.js | 4 +- 7 files changed, 163 insertions(+), 265 deletions(-) create mode 100755 utils/flakiness-dashboard/processing/compress_raw_data.js create mode 100644 utils/flakiness-dashboard/processing/dashboard_compressed_v1.js delete mode 100644 utils/flakiness-dashboard/processing/dashboard_v1.js delete mode 100644 utils/flakiness-dashboard/processing/dashboard_v2.js diff --git a/utils/flakiness-dashboard/processing/compress_raw_data.js b/utils/flakiness-dashboard/processing/compress_raw_data.js new file mode 100755 index 0000000000000..3d20ed4e0fff1 --- /dev/null +++ b/utils/flakiness-dashboard/processing/compress_raw_data.js @@ -0,0 +1,34 @@ +#!/usr/bin/env node +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); +const fs = require('fs'); +const {SimpleBlob} = require('./utils.js'); +const {processDashboardCompressedV1} = require('./dashboard_compressed_v1.js'); + +(async () => { + const sha = process.argv[2]; + console.log(sha); + const dashboardBlob = await SimpleBlob.create('dashboards', `raw/${sha}.json`); + const reports = await dashboardBlob.download(); + if (!reports) { + console.error('ERROR: no data found for commit ' + sha); + process.exit(1); + } + await processDashboardCompressedV1({log: console.log}, reports, sha); +})(); diff --git a/utils/flakiness-dashboard/processing/dashboard_compressed_v1.js b/utils/flakiness-dashboard/processing/dashboard_compressed_v1.js new file mode 100644 index 0000000000000..83cf5a2f41ce2 --- /dev/null +++ b/utils/flakiness-dashboard/processing/dashboard_compressed_v1.js @@ -0,0 +1,119 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the 'License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {SimpleBlob, flattenSpecs} = require('./utils.js'); + +async function processDashboardCompressedV1(context, reports, commitSHA) { + const timestamp = Date.now(); + const dashboardBlob = await SimpleBlob.create('dashboards', `compressed_v1/${commitSHA}.json`); + await dashboardBlob.uploadGzipped(compressReports(reports)); + + context.log(` + ===== started dashboard compressed v1 ===== + SHA: ${commitSHA} + ===== complete in ${Date.now() - timestamp}ms ===== + `); +} + +module.exports = {processDashboardCompressedV1, compressReports}; + +function compressReports(reports) { + const files = {}; + for (const report of reports) { + for (const spec of flattenSpecs(report)) { + let specs = files[spec.file]; + if (!specs) { + specs = new Map(); + files[spec.file] = specs; + } + const specId = spec.file + '---' + spec.title + ' --- ' + spec.line; + let specObject = specs.get(specId); + if (!specObject) { + specObject = { + title: spec.title, + line: spec.line, + column: spec.column, + tests: new Map(), + }; + specs.set(specId, specObject); + } + for (const test of spec.tests || []) { + if (test.runs.length === 1 && !test.runs[0].status) + continue; + // Overwrite test platform parameter with a more specific information from + // build run. + const osName = report.metadata.osName.toUpperCase().startsWith('MINGW') ? 'Windows' : report.metadata.osName; + const arch = report.metadata.arch && !report.metadata.arch.includes('x86') ? report.metadata.arch : ''; + const platform = (osName + ' ' + report.metadata.osVersion + ' ' + arch).trim(); + const browserName = test.parameters.browserName || 'N/A'; + + const testName = getTestName(browserName, platform, test.parameters); + let testObject = specObject.tests.get(testName); + if (!testObject) { + testObject = { + parameters: { + ...test.parameters, + browserName, + platform, + }, + }; + // By default, all tests are expected to pass. We can have this as a hidden knowledge. + if (test.expectedStatus !== 'passed') + testObject.expectedStatus = test.expectedStatus; + if (test.annotations.length) + testObject.annotations = test.annotations; + specObject.tests.set(testName, testObject); + } + + for (const run of test.runs) { + // Record duration of slow tests only, i.e. > 1s. + if (run.status === 'passed' && run.duration > 1000) { + testObject.minTime = Math.min((testObject.minTime || Number.MAX_VALUE), run.duration); + testObject.maxTime = Math.max((testObject.maxTime || 0), run.duration); + } + if (run.status === 'failed') { + if (!Array.isArray(testObject.failed)) + testObject.failed = []; + testObject.failed.push(run.error); + } else { + testObject[run.status] = (testObject[run.status] || 0) + 1; + } + } + } + } + } + + const pojo = Object.entries(files).map(([file, specs]) => ({ + file, + specs: [...specs.values()].map(specObject => ({ + ...specObject, + tests: [...specObject.tests.values()], + })), + })); + return pojo; +} + +function getTestName(browserName, platform, parameters) { + return [browserName, platform, ...Object.entries(parameters).filter(([key, value]) => !!value).map(([key, value]) => { + if (key === 'browserName' || key === 'platform') + return; + if (typeof value === 'string') + return value; + if (typeof value === 'boolean') + return key; + return `${key}=${value}`; + }).filter(Boolean)].join(' / '); +} diff --git a/utils/flakiness-dashboard/processing/dashboard_raw.js b/utils/flakiness-dashboard/processing/dashboard_raw.js index 8c2059987382a..3d250972762ea 100644 --- a/utils/flakiness-dashboard/processing/dashboard_raw.js +++ b/utils/flakiness-dashboard/processing/dashboard_raw.js @@ -30,6 +30,10 @@ async function processDashboardRaw(context, report) { timestamp: ${report.metadata.commitTimestamp} ===== complete in ${Date.now() - timestamp}ms ===== `); + return { + reports: dashboardData, + commitSHA: report.metadata.commitSHA, + }; } module.exports = {processDashboardRaw}; diff --git a/utils/flakiness-dashboard/processing/dashboard_v1.js b/utils/flakiness-dashboard/processing/dashboard_v1.js deleted file mode 100644 index 5c6e5a79a8554..0000000000000 --- a/utils/flakiness-dashboard/processing/dashboard_v1.js +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the 'License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {SimpleBlob, flattenSpecs} = require('./utils.js'); - -const DASHBOARD_VERSION = 1; - -class Dashboard { - constructor() { - this._runs = []; - } - - initialize(jsonData) { - if (jsonData.version !== DASHBOARD_VERSION) { - // Run migrations here! - } - this._runs = jsonData.buildbotRuns; - } - - addReport(report) { - // We cannot use linenumber to identify specs since line numbers - // might be different across commits. - const getSpecId = spec => spec.file + ' @@@ ' + spec.title; - - const faultySpecIds = new Set(); - for (const run of this._runs) { - for (const spec of run.specs) - faultySpecIds.add(getSpecId(spec)); - } - const specs = []; - for (const spec of flattenSpecs(report)) { - // Filter out specs that didn't have a single test that was run in the - // given shard. - if (spec.tests.every(test => test.runs.length === 1 && !test.runs[0].status)) - continue; - const hasFlakyAnnotation = spec.tests.some(test => test.annotations.some(a => a.type === 'flaky')); - - if (!spec.ok || hasFlakyAnnotation || faultySpecIds.has(getSpecId(spec))) - specs.push(spec); - } - if (specs.length) { - this._runs.push({ - metadata: report.metadata, - specs, - }); - } - return specs.length; - } - - serialize(maxCommits = 100) { - const shaToTimestamp = new Map(); - for (const run of this._runs) - shaToTimestamp.set(run.metadata.commitSHA, run.metadata.commitTimestamp); - const commits = [...shaToTimestamp].sort(([sha1, ts1], [sha2, ts2]) => ts2 - ts1).slice(0, maxCommits); - const commitsSet = new Set(commits.map(([sha, ts]) => sha)); - return { - version: DASHBOARD_VERSION, - timestamp: Date.now(), - buildbotRuns: this._runs.filter(run => commitsSet.has(run.metadata.commitSHA)), - }; - } -} - -async function processDashboardV1(context, report) { - const timestamp = Date.now(); - const dashboardBlob = await SimpleBlob.create('dashboards', 'main.json'); - const dashboardData = await dashboardBlob.download(); - const dashboard = new Dashboard(); - if (dashboardData) - dashboard.initialize(dashboardData); - - try { - const addedSpecs = dashboard.addReport(report); - await dashboardBlob.uploadGzipped(dashboard.serialize()); - context.log(` - ===== started dashboard v1 ===== - SHA: ${report.metadata.commitSHA} - URL: ${report.metadata.runURL} - timestamp: ${report.metadata.commitTimestamp} - added specs: ${addedSpecs} - ===== complete in ${Date.now() - timestamp}ms ===== - `); - } catch (e) { - context.log(e); - return; - } -} - -module.exports = {processDashboardV1}; diff --git a/utils/flakiness-dashboard/processing/dashboard_v2.js b/utils/flakiness-dashboard/processing/dashboard_v2.js deleted file mode 100644 index 6211b95670251..0000000000000 --- a/utils/flakiness-dashboard/processing/dashboard_v2.js +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the 'License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {SimpleBlob, flattenSpecs} = require('./utils.js'); - -const DASHBOARD_VERSION = 1; - -class Dashboard { - constructor() { - this._specs = new Map(); - this._commits = new Map(); - } - - initialize(jsonData) { - if (jsonData.version !== DASHBOARD_VERSION) { - // Run migrations here! - } - for (const spec of jsonData.specs) { - const commitCoordinates = new Map(); - for (const coord of spec.commitCoordinates) - commitCoordinates.set(coord.sha, coord); - this._specs.set(spec.specId, { - specId: spec.specId, - file: spec.file, - title: spec.title, - problematicTests: spec.problematicTests, - commitCoordinates, - }); - } - for (const commit of jsonData.commits) - this._commits.set(commit.sha, commit); - } - - addReport(report) { - const sha = report.metadata.commitSHA; - this._commits.set(sha, { - sha, - timestamp: report.metadata.commitTimestamp, - message: report.metadata.commitTitle, - author: report.metadata.commitAuthorName, - email: report.metadata.commitAuthorEmail, - }); - let addedSpecs = 0; - for (const spec of flattenSpecs(report)) { - // We cannot use linenumber to identify specs since line numbers - // might be different across commits. - const specId = spec.file + ' --- ' + spec.title; - const tests = spec.tests.filter(test => !isHealthyTest(test)); - // If there are no problematic testruns now and before - ignore the spec. - if (!tests.length && !this._specs.has(specId)) - continue; - ++addedSpecs; - let specInfo = this._specs.get(specId); - if (!specInfo) { - specInfo = { - specId, - title: spec.title, - file: spec.file, - commitCoordinates: new Map(), - problematicTests: [], - }; - this._specs.set(specId, specInfo); - } - specInfo.problematicTests.push(...tests.map(test => ({sha, test}))); - specInfo.commitCoordinates.set(sha, ({sha, line: spec.line, column: spec.column})); - } - return addedSpecs; - } - - serialize(maxCommits = 100) { - const commits = [...this._commits.values()].sort((a, b) => a.timestamp - b.timestamp).slice(-maxCommits); - const whitelistedCommits = new Set(); - for (const c of commits) - whitelistedCommits.add(c.sha); - - const specs = [...this._specs.values()].map(spec => ({ - specId: spec.specId, - title: spec.title, - file: spec.file, - commitCoordinates: [...spec.commitCoordinates.values()].filter(coord => whitelistedCommits.has(coord.sha)), - problematicTests: [...spec.problematicTests.values()].filter(test => whitelistedCommits.has(test.sha)), - })).filter(spec => spec.commitCoordinates.length && spec.problematicTests.length); - - return { - version: DASHBOARD_VERSION, - timestamp: Date.now(), - commits, - specs, - }; - } -} - -async function processDashboardV2(context, report) { - const timestamp = Date.now(); - const dashboardBlob = await SimpleBlob.create('dashboards', 'main_v2.json'); - const dashboardData = await dashboardBlob.download(); - const dashboard = new Dashboard(); - if (dashboardData) - dashboard.initialize(dashboardData); - - try { - const addedSpecs = dashboard.addReport(report); - await dashboardBlob.uploadGzipped(dashboard.serialize()); - context.log(` - ===== started dashboard v2 ===== - SHA: ${report.metadata.commitSHA} - URL: ${report.metadata.runURL} - timestamp: ${report.metadata.commitTimestamp} - added specs: ${addedSpecs} - ===== complete in ${Date.now() - timestamp}ms ===== - `); - } catch (e) { - context.log(e); - return; - } -} - -module.exports = {processDashboardV2}; - -function isHealthyTest(test) { - // If test has any annotations - it's not healthy and requires attention. - if (test.annotations.length) - return false; - // If test does not have annotations and doesn't have runs - it's healthy. - if (!test.runs.length) - return true; - // If test was run more than once - it's been retried and thus unhealthy. - if (test.runs.length > 1) - return false; - const run = test.runs[0]; - // Test might not have status if it was sharded away - consider it healthy. - if (!run.status) - return true; - // if status is not "passed", then it's a bad test. - if (run.status !== 'passed') - return false; - // if run passed, but that's not what we expected - it's a bad test. - if (run.status !== test.expectedStatus) - return false; - // Otherwise, the test is healthy. - return true; -} diff --git a/utils/flakiness-dashboard/processing/index.js b/utils/flakiness-dashboard/processing/index.js index dc1c500e0627f..c25e46ba5a87d 100644 --- a/utils/flakiness-dashboard/processing/index.js +++ b/utils/flakiness-dashboard/processing/index.js @@ -15,9 +15,8 @@ */ const {blobServiceClient, gunzipAsync, deleteBlob} = require('./utils.js'); -const {processDashboardV1} = require('./dashboard_v1.js'); -const {processDashboardV2} = require('./dashboard_v2.js'); const {processDashboardRaw} = require('./dashboard_raw.js'); +const {processDashboardCompressedV1} = require('./dashboard_compressed_v1.js'); module.exports = async function(context) { // First thing we do - delete the blob. @@ -28,9 +27,6 @@ module.exports = async function(context) { const report = JSON.parse(data.toString('utf8')); // Process dashboards one-by-one to limit max heap utilization. - await processDashboardRaw(context, report); - // Disable V1 dashboard since it's crazy expensive to compute. - // await processDashboardV1(context, report); - // Disable V2 dashboard in favor of raw data. - // await processDashboardV2(context, report); + const {reports, commitSHA} = await processDashboardRaw(context, report); + await processDashboardCompressedV1(context, reports, commitSHA); } diff --git a/utils/flakiness-dashboard/processing/utils.js b/utils/flakiness-dashboard/processing/utils.js index 4f3e95b9450dd..d0e8dc6204386 100644 --- a/utils/flakiness-dashboard/processing/utils.js +++ b/utils/flakiness-dashboard/processing/utils.js @@ -61,7 +61,9 @@ class SimpleBlob { async uploadGzipped(data) { const content = JSON.stringify(data); - const zipped = await gzipAsync(content); + const zipped = await gzipAsync(content, { + level: 9, + }); await this._blockBlobClient.upload(zipped, Buffer.byteLength(zipped), { blobHTTPHeaders: { blobContentEncoding: 'gzip',