-
Notifications
You must be signed in to change notification settings - Fork 9.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
core(proto): add proto definition for LHR (#6183)
- Loading branch information
Showing
11 changed files
with
4,083 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| /** | ||
| * @license Copyright 2018 Google Inc. All Rights Reserved. | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
| */ | ||
|
|
||
| 'use strict'; | ||
|
|
||
| const fs = require('fs'); | ||
|
|
||
| /** | ||
| * @fileoverview Helper functions to transform an LHR into a proto-ready LHR. | ||
| * | ||
| * FIXME: This file is 100% technical debt. Our eventual goal is for the | ||
| * roundtrip JSON to match the Golden LHR 1:1. | ||
| */ | ||
|
|
||
| /** | ||
| * @param {string} result | ||
| */ | ||
| function processForProto(result) { | ||
| /** @type {LH.Result} */ | ||
| const reportJson = JSON.parse(result); | ||
|
|
||
| // Clean up actions that require 'audits' to exist | ||
| if ('audits' in reportJson) { | ||
| Object.keys(reportJson.audits).forEach(auditName => { | ||
| const audit = reportJson.audits[auditName]; | ||
|
|
||
| // Rewrite the 'not-applicable' scoreDisplayMode to 'not_applicable'. #6201 | ||
| if ('scoreDisplayMode' in audit) { | ||
| if (audit.scoreDisplayMode === 'not-applicable') { | ||
| // @ts-ignore Breaking the LH.Result type | ||
| audit.scoreDisplayMode = 'not_applicable'; | ||
| } | ||
| } | ||
| // Drop raw values. #6199 | ||
| if ('rawValue' in audit) { | ||
| delete audit.rawValue; | ||
| } | ||
| // Normalize displayValue to always be a string, not an array. #6200 | ||
|
|
||
| if (Array.isArray(audit.displayValue)) { | ||
| /** @type {Array<any>}*/ | ||
| const values = []; | ||
| audit.displayValue.forEach(item => { | ||
| values.push(item); | ||
| }); | ||
| audit.displayValue = values.join(' | '); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| // Drop the i18n icuMessagePaths. Painful in proto, and low priority to expose currently. | ||
| if ('i18n' in reportJson && 'icuMessagePaths' in reportJson.i18n) { | ||
| delete reportJson.i18n.icuMessagePaths; | ||
| } | ||
|
|
||
| // Remove any found empty strings, as they are dropped after round-tripping anyway | ||
| /** | ||
| * @param {any} obj | ||
| */ | ||
| function removeStrings(obj) { | ||
| if (obj && typeof obj === 'object' && !Array.isArray(obj)) { | ||
| Object.keys(obj).forEach(key => { | ||
| if (typeof obj[key] === 'string' && obj[key] === '') { | ||
| delete obj[key]; | ||
| } else if (typeof obj[key] === 'object' || Array.isArray(obj[key])) { | ||
| removeStrings(obj[key]); | ||
| } | ||
| }); | ||
| } else if (Array.isArray(obj)) { | ||
| obj.forEach(item => { | ||
| if (typeof item === 'object' || Array.isArray(item)) { | ||
| removeStrings(item); | ||
| } | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| removeStrings(reportJson); | ||
|
|
||
| return JSON.stringify(reportJson); | ||
| } | ||
|
|
||
| // @ts-ignore claims always false, but this checks if cli or module | ||
| if (require.main === module) { | ||
| // read in the argv for the input & output | ||
| const args = process.argv.slice(2); | ||
| let input; | ||
| let output; | ||
|
|
||
| if (args.length) { | ||
| // find can return undefined, so default it to '' with OR | ||
| input = (args.find(flag => flag.startsWith('--in')) || '').replace('--in=', ''); | ||
| output = (args.find(flag => flag.startsWith('--out')) || '').replace('--out=', ''); | ||
| } | ||
|
|
||
| if (input && output) { | ||
| // process the file | ||
| const report = processForProto(fs.readFileSync(input, 'utf-8')); | ||
| // write to output from argv | ||
| fs.writeFileSync(output, report, 'utf-8'); | ||
| } | ||
| } else { | ||
| module.exports = { | ||
| processForProto, | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| /** | ||
| * @license Copyright 2018 Google Inc. All Rights Reserved. | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
| */ | ||
| 'use strict'; | ||
|
|
||
| const processForProto = require('../../lib/proto-preprocessor').processForProto; | ||
|
|
||
| /* eslint-env jest */ | ||
| describe('processing for proto', () => { | ||
| it('cleans up audits', () => { | ||
| const input = { | ||
| 'audits': { | ||
| 'critical-request-chains': { | ||
| 'scoreDisplayMode': 'not-applicable', | ||
| 'rawValue': 14.3, | ||
| 'displayValue': ['hello %d', 123], | ||
| }, | ||
| }, | ||
| }; | ||
| const expectation = { | ||
| 'audits': { | ||
| 'critical-request-chains': { | ||
| 'scoreDisplayMode': 'not_applicable', | ||
| 'displayValue': 'hello %d | 123', | ||
| }, | ||
| }, | ||
| }; | ||
| const output = processForProto(JSON.stringify(input)); | ||
|
|
||
| expect(JSON.parse(output)).toMatchObject(expectation); | ||
| }); | ||
|
|
||
|
|
||
| it('removes i18n icuMessagePaths', () => { | ||
| const input = { | ||
| 'i18n': { | ||
| 'icuMessagePaths': { | ||
| 'content': 'paths', | ||
| }, | ||
| }, | ||
| }; | ||
| const expectation = { | ||
| 'i18n': {}, | ||
| }; | ||
| const output = processForProto(JSON.stringify(input)); | ||
|
|
||
| expect(JSON.parse(output)).toMatchObject(expectation); | ||
| }); | ||
|
|
||
| it('removes empty strings', () => { | ||
| const input = { | ||
| 'audits': { | ||
| 'critical-request-chains': { | ||
| 'details': { | ||
| 'chains': { | ||
| '1': '', | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| 'i18n': { | ||
| 'icuMessagePaths': { | ||
| 'content': 'paths', | ||
| }, | ||
| '2': '', | ||
| '3': [ | ||
| { | ||
| 'hello': 'world', | ||
| '4': '', | ||
| }, | ||
| ], | ||
| }, | ||
| }; | ||
| const expectation = { | ||
| 'audits': { | ||
| 'critical-request-chains': { | ||
| 'details': { | ||
| 'chains': {}, | ||
| }, | ||
| }, | ||
| }, | ||
| 'i18n': { | ||
| '3': [ | ||
| {'hello': 'world'}, | ||
| ], | ||
| }, | ||
| }; | ||
| const output = processForProto(JSON.stringify(input)); | ||
|
|
||
| expect(JSON.parse(output)).toMatchObject(expectation); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| /** | ||
| * @license Copyright 2017 Google Inc. All Rights Reserved. | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||
| */ | ||
| 'use strict'; | ||
|
|
||
| const path = require('path'); | ||
| const fs = require('fs'); | ||
|
|
||
| const sample = fs.readFileSync(path.resolve(__dirname, '../results/sample_v2.json')); | ||
| const roundTripJson = require('../../../proto/sample_v2_round_trip'); | ||
| const preprocessor = require('../../lib/proto-preprocessor.js'); | ||
|
|
||
| /* eslint-env jest */ | ||
|
|
||
| describe('round trip JSON comparison subsets', () => { | ||
| let sampleJson; | ||
|
|
||
| beforeEach(() => { | ||
| sampleJson = JSON.parse(preprocessor.processForProto(sample)); | ||
| }); | ||
|
|
||
| it('has the same audit results sans details', () => { | ||
| Object.keys(sampleJson.audits).forEach(audit => { | ||
| delete sampleJson.audits[audit].details; | ||
| }); | ||
|
|
||
| expect(roundTripJson.audits).toMatchObject(sampleJson.audits); | ||
| }); | ||
|
|
||
| it('has the same audit results & details if applicable', () => { | ||
| Object.keys(sampleJson.audits).forEach(auditId => { | ||
| expect(roundTripJson.audits[auditId]).toMatchObject(sampleJson.audits[auditId]); | ||
|
|
||
| if ('details' in sampleJson.audits[auditId]) { | ||
| expect(roundTripJson.audits[auditId].details) | ||
| .toMatchObject(sampleJson.audits[auditId].details); | ||
| } | ||
| }); | ||
| }); | ||
|
|
||
| it('has the same i18n rendererFormattedStrings', () => { | ||
| expect(roundTripJson.i18n).toMatchObject(sampleJson.i18n); | ||
| }); | ||
|
|
||
| it('has the same top level values', () => { | ||
| Object.keys(sampleJson).forEach(audit => { | ||
| if (typeof sampleJson[audit] === 'object' && !Array.isArray(sampleJson[audit])) { | ||
| delete sampleJson[audit]; | ||
| } | ||
| }); | ||
|
|
||
| expect(roundTripJson).toMatchObject(sampleJson); | ||
| }); | ||
|
|
||
| it('has the same config values', () => { | ||
| expect(roundTripJson.configSettings).toMatchObject(sampleJson.configSettings); | ||
| }); | ||
| }); | ||
|
|
||
| describe('round trip JSON comparison to everything', () => { | ||
| let sampleJson; | ||
|
|
||
| beforeEach(() => { | ||
| sampleJson = JSON.parse(preprocessor.processForProto(sample)); | ||
| }); | ||
|
|
||
| it('has the same JSON overall', () => { | ||
| expect(roundTripJson).toMatchObject(sampleJson); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| ## How to compile protos + use locally | ||
|
|
||
| 1. Install the proto compiler | ||
| 1. Manual install | ||
| 1. Get the latest proto [release](https://github.com/protocolbuffers/protobuf/releases) (select one with python included if you want to run this validator) | ||
| 1. Install the [C++ Protocol Buffer Runtime](https://github.com/protocolbuffers/protobuf/blob/master/src/README.md) | ||
| 1. Brew install | ||
| 1. `brew install protobuf` | ||
| 1. Run `yarn compile-proto` then `yarn build-proto` | ||
|
|
||
| ## Proto Resources | ||
| - [Protobuf Github Repo](https://github.com/protocolbuffers/protobuf) | ||
| - [Protobuf Docs](https://developers.google.com/protocol-buffers/docs/overview) | ||
|
|
||
| ## LHR Round Trip Flow | ||
| ``` | ||
| LHR round trip flow: | ||
| (Compiling the Proto) | ||
| lighthouse_result.proto -> protoc --python_out ... -> lighthouse_result.pb2 | ||
| ⭏ | ||
| (used by) | ||
| (Making a Round Trip JSON) ⭏ | ||
| lhr.json --> proto_preprocessor.js -> lhr_processed.json -> json_roundtrip_via_proto.py -> lhr.round_trip.json | ||
| ``` | ||
|
|
||
| ## Hacking Hints | ||
| - Clean out compiled proto and json with `yarn clean` | ||
| - Round trips might jumble the order of your JSON keys and lists! | ||
| - Is your `six` installation troubling your `pip install protobuf`? Mine did. Try `pip install --ignore-installed six`. |
Oops, something went wrong.