feat: replaced chrome 80, 81, 83 emulators with more robust os-level …

…data
ulixee · Feb 5, 2021 · 276b269 · 276b269
1 parent 7066543
commit 276b269
Show file tree

Hide file tree

Showing 804 changed files with 173,499 additions and 1,549,996 deletions.
diff --git a/copy-build.js b/copy-build.js
@@ -5,6 +5,7 @@ const Fs = require('fs');
 copyfiles(
   [
     'emulate-browsers/**/data/**',
+    'emulate-browsers/**/config.json',
     'testing/*/**',
     'examples/*.mjs',
     '*client/index.mjs',

diff --git a/copy-dist.js b/copy-dist.js
@@ -9,6 +9,7 @@ copyfiles(
     '-e "build-dist/**"',
     '-e "mitm-socket/dist"',
     'emulate-browsers/**/data/**',
+    'emulate-browsers/**/data/config.json',
     'mitm-socket/lib/*',
     '**/.gitignore',
     '**/.npmignore',

diff --git a/core-interfaces/IBrowserEmulator.ts b/core-interfaces/IBrowserEmulator.ts
@@ -12,6 +12,7 @@ export default interface IBrowserEmulator {
   readonly windowFraming?: IWindowFraming;
   locale: string;
   userProfile: IUserProfile;
+  sessionId?: string;
 
   newDocumentInjectedScripts(): Promise<INewDocumentInjectedScript[]>;
 }
diff --git a/core/lib/BrowserEmulators.ts b/core/lib/BrowserEmulators.ts
@@ -23,14 +23,18 @@ export default class BrowserEmulators {
   public static getClass(browserEmulatorId: string) {
     let BrowserEmulator = this.emulatorsById[browserEmulatorId];
     if (!BrowserEmulator) {
-      const fromShortId = `@secret-agent/emulate-${browserEmulatorId}`;
       try {
+        const fromShortId = `@secret-agent/emulate-${browserEmulatorId}`;
         // eslint-disable-next-line global-require,import/no-dynamic-require
         BrowserEmulator = require(fromShortId)?.default;
       } catch (err) {
-        // try as full package name
-        // eslint-disable-next-line global-require,import/no-dynamic-require
-        BrowserEmulator = require(browserEmulatorId)?.default;
+        try {
+          // try as full package name
+          // eslint-disable-next-line global-require,import/no-dynamic-require
+          BrowserEmulator = require(browserEmulatorId)?.default;
+        } catch (e) {
+          // silently fail
+        }
       }
       if (BrowserEmulator) this.load(BrowserEmulator);
     }

diff --git a/core/lib/Session.ts b/core/lib/Session.ts
@@ -80,6 +80,7 @@ export default class Session extends TypedEventEmitter<{
     if (options.userProfile) {
       this.userProfile = options.userProfile;
       this.browserEmulator.userProfile = options.userProfile;
+      this.browserEmulator.sessionId = this.id;
     }
     this.upstreamProxyUrl = options.upstreamProxyUrl;
 

diff --git a/emulate-browsers/base/data/darwinToMacOsVersionMap.json b/emulate-browsers/base/data/darwinToMacOsVersionMap.json
@@ -26,5 +26,8 @@
   "18.2.0": "10.14.1",
   "19.2.0": "10.15.2",
   "19.3.0": "10.15.3",
-  "19.5.0": "10.15.5"
+  "20.0.0": "11.0",
+  "20.1.0": "11.0",
+  "20.2.0": "11.1",
+  "20.3.0": "11.2"
 }
diff --git a/emulate-browsers/base/data/macOsVersionAliasMap.json b/emulate-browsers/base/data/macOsVersionAliasMap.json
@@ -0,0 +1,5 @@
+{
+  "10.16": "11",
+  "11.0": "11",
+  "11.1": "11"
+}
diff --git a/emulate-browsers/base/injected-scripts/_descriptorBuilder.ts b/emulate-browsers/base/injected-scripts/_descriptorBuilder.ts
@@ -26,8 +26,8 @@ function createError(message: string, type?: { new (message: string): any }) {
 
 function newObjectConstructor(newProps: IDescriptor) {
   return function() {
-    if (typeof newProps === 'string') {
-      throw createError(newProps);
+    if (newProps._$constructorException) {
+      throw createError(newProps._$constructorException);
     }
     Object.setPrototypeOf(this, getObjectAtPath(newProps._$protos[0]));
     const props = Object.entries(newProps);
@@ -176,6 +176,7 @@ declare interface IDescriptor {
   _$get?: any;
   _$set?: any;
   _$accessException?: string;
+  _$constructorException?: string;
   _$value?: string;
   '_$$value()'?: () => string;
   _$function?: string;

diff --git a/emulate-browsers/base/lib/DataLoader.ts b/emulate-browsers/base/lib/DataLoader.ts
@@ -3,7 +3,7 @@ import * as Fs from 'fs';
 export default class DataLoader {
   private readonly dataDir: string;
   private readonly type: string;
-  private dataMap: {} = {};
+  private dataMap: { [operatingSystemId: string]: any } = {};
 
   constructor(dataDir: string, type: string) {
     this.dataDir = dataDir;
@@ -12,7 +12,7 @@ export default class DataLoader {
 
   public get(operatingSystemId: string) {
     if (!this.dataMap[operatingSystemId]) {
-      const path = `${this.dataDir}/${operatingSystemId}/${this.type}.json`;
+      const path = `${this.dataDir}/as-${operatingSystemId}/${this.type}.json`;
       const data = JSON.parse(Fs.readFileSync(path, 'utf8'));
       this.dataMap[operatingSystemId] = data;
     }

diff --git a/emulate-browsers/base/lib/DomDiffLoader.ts b/emulate-browsers/base/lib/DomDiffLoader.ts
@@ -14,7 +14,7 @@ export default class DomDiffLoader {
 
   public get(operatingSystemId: string) {
     if (!this.dataMap[operatingSystemId]) {
-      const dir = `${this.dataDir}/${operatingSystemId}`;
+      const dir = `${this.dataDir}/as-${operatingSystemId}`;
       const filename = extractFilename(Fs.readdirSync(dir));
       const data = JSON.parse(Fs.readFileSync(`${dir}/${filename}`, 'utf8'));
       this.dataMap[operatingSystemId] = data;
@@ -28,7 +28,7 @@ function extractFilename(filenames: string[]) {
   const localOsVersion = localOsMeta.version;
   const filenameMap = {};
   for (const filename of filenames) {
-    const matches = filename.match(/^dom-diffs-when-using-([a-z-]+)(-([0-9-]+))?.json$/);
+    const matches = filename.match(/^dom-diffs-when-runtime-([a-z-]+)(-([0-9-]+))?.json$/);
     if (!matches) continue;
 
     const [osName, _, osVersion] = matches.slice(1); // eslint-disable-line @typescript-eslint/naming-convention,@typescript-eslint/no-unused-vars
@@ -57,16 +57,23 @@ function findClosestVersionMatch(versionToMatch: string, versions: string[]) {
 
   // there is no guarantee we have an exact match, so let's get the closest
   const versionTree = convertVersionsToTree(versions);
-  const [major, minor] = versionToMatch.split('-');
+  const [major, minor] = versionToMatch.split('-').map(x => Number(x));
 
   const majors = Object.keys(versionTree).map(x => Number(x));
-  const majorMatch = getClosestNumberMatch(Number(major), majors);
+  const majorMatch = getClosestNumberMatch(major, majors);
   let versionMatch = `${majorMatch}`;
 
   if (minor) {
     const minors = Object.keys(versionTree[majorMatch]).map(x => Number(x));
-    const minorMatch = getClosestNumberMatch(Number(minor), minors);
+    const minorMatch = getClosestNumberMatch(minor, minors);
     versionMatch += `-${minorMatch}`;
+  } else if (!versions.includes(versionMatch)) {
+    const minors = Object.keys(versionTree[majorMatch]).map(x => Number(x));
+    if (minors.length) {
+      const minorMatch = major > majorMatch ? Math.max(...minors) : Math.min(...minors);
+      versionMatch += `-${minorMatch}`;
+    }
   }
+
   return versions.includes(versionMatch) ? versionMatch : null;
 }
diff --git a/emulate-browsers/base/lib/getLocalOperatingSystemMeta.ts b/emulate-browsers/base/lib/getLocalOperatingSystemMeta.ts
@@ -1,6 +1,7 @@
 import Os from 'os';
-import { convertVersionsToTree, getClosestNumberMatch } from "./VersionUtils";
+import { convertVersionsToTree, getClosestNumberMatch } from './VersionUtils';
 import darwinToMacOsVersionMap from '../data/darwinToMacOsVersionMap.json';
+import macOsVersionAliasMap from '../data/macOsVersionAliasMap.json';
 import windowsToWindowsVersionMap from '../data/windowsToWindowsVersionMap.json';
 
 export default function getLocalOperatingSystemMeta() {
@@ -58,7 +59,8 @@ function extractMacOsVersion(release: string) {
     versionMatch += `.${buildMatch}`;
   }
 
-  return darwinToMacOsVersionMap[versionMatch];
+  const versionString = darwinToMacOsVersionMap[versionMatch];
+  return macOsVersionAliasMap[versionString] || versionString;
 }
 
 const platformToOsName = {
@@ -69,8 +71,7 @@ const platformToOsName = {
   freebsd: 'linux',
   openbsd: 'linux',
   sunos: 'linux',
-}
+};
 
 const darwinVersionTree = convertVersionsToTree(Object.keys(darwinToMacOsVersionMap));
 const windowsVersionTree = convertVersionsToTree(Object.keys(windowsToWindowsVersionMap));
-
diff --git a/emulate-browsers/base/scripts/updateData.ts b/emulate-browsers/base/scripts/updateData.ts
@@ -6,7 +6,9 @@ const slabBasicsDir = Path.join(slabDataDir, 'basics');
 const dataDir = Path.resolve(__dirname, '../data');
 
 const darwinToMacOsVersionMap = Fs.readFileSync(`${slabBasicsDir}/darwinToMacOsVersionMap.json`, 'utf8');
+const macOsVersionAliasMap = Fs.readFileSync(`${slabBasicsDir}/macOsVersionAliasMap.json`, 'utf8');
 const windowsToWindowsVersionMap = Fs.readFileSync(`${slabBasicsDir}/windowsToWindowsVersionMap.json`, 'utf8');
 
 Fs.writeFileSync(`${dataDir}/darwinToMacOsVersionMap.json`, darwinToMacOsVersionMap);
+Fs.writeFileSync(`${dataDir}/macOsVersionAliasMap.json`, macOsVersionAliasMap);
 Fs.writeFileSync(`${dataDir}/windowsToWindowsVersionMap.json`, windowsToWindowsVersionMap);
diff --git a/emulate-browsers/base/test/chrome.test.ts b/emulate-browsers/base/test/chrome.test.ts
@@ -1,5 +1,5 @@
 import * as Helpers from '@secret-agent/testing/helpers';
-import windowChrome from '@secret-agent/emulate-chrome-80/data/mac-os-10-14/window-chrome.json';
+import windowChrome from '@secret-agent/emulate-chrome-80/data/as-mac-os-10-14/window-chrome.json';
 import { inspect } from 'util';
 import BrowserEmulators from '@secret-agent/core/lib/BrowserEmulators';
 import { GlobalPool } from '@secret-agent/core';

diff --git a/emulate-browsers/base/test/plugins.test.ts b/emulate-browsers/base/test/plugins.test.ts
@@ -4,7 +4,7 @@ import Puppet from '@secret-agent/puppet';
 import { GlobalPool } from '@secret-agent/core';
 import BrowserEmulators from '@secret-agent/core/lib/BrowserEmulators';
 import Log from '@secret-agent/commons/Logger';
-import navigatorJson from '@secret-agent/emulate-chrome-80/data/mac-os-10-14/window-navigator.json';
+import navigatorJson from '@secret-agent/emulate-chrome-80/data/as-mac-os-10-14/window-navigator.json';
 import pluginsChrome from './plugins-Chrome.json';
 import inspectScript from './inspectHierarchy';
 import { getOverrideScript } from '../lib/DomOverridesBuilder';

diff --git a/emulate-browsers/chrome-80/config.json b/emulate-browsers/chrome-80/config.json
@@ -0,0 +1,10 @@
+{
+  "browserId": "chrome-80-0",
+  "browserEngine": {
+    "id": "chromium-80-0",
+    "name": "chromium",
+    "revision": "722234",
+    "features": ["headless"]
+  },
+  "marketshare": 0.4
+}
diff --git a/...ome-80/data/mac-os-10-13/clienthello.json → ...-80/data/as-mac-os-10-10/clienthello.json b/...ome-80/data/mac-os-10-13/clienthello.json → ...-80/data/as-mac-os-10-10/clienthello.json
@@ -1,7 +1,7 @@
 {
   "version": "0x303 (TLS 1.2)",
   "ciphers": [
-    "{0xAA, 0xAA} GREASE",
+    "{0xBA, 0xBA} GREASE",
     "{0x13, 0x01} TLS_AES_128_GCM_SHA256",
     "{0x13, 0x02} TLS_AES_256_GCM_SHA384",
     "{0x13, 0x03} TLS_CHACHA20_POLY1305_SHA256",
@@ -23,7 +23,7 @@
   "extensions": [
     {
       "type": "GREASE",
-      "decimal": 60138,
+      "decimal": 23130,
       "values": []
     },
     {
@@ -101,7 +101,7 @@
         "NamedGroup: GREASE (35466)",
         "key_exchange:  (len=1): 00",
         "NamedGroup: ecdh_x25519 (29)",
-        "key_exchange:  (len=32): F0269F11516AD17E1B8C3DA2F8222C9A0856A82BE7A9E0A65C05C3AEF7A66651"
+        "key_exchange:  (len=32): 9B20FBB462B06D3513198852DDAFAA02462DDB9C7597A67F955E356BD99CDD76"
       ]
     },
     {
@@ -113,7 +113,7 @@
       "type": "supported_versions",
       "decimal": 43,
       "values": [
-        "GREASE (6682)",
+        "GREASE (27242)",
         "TLS 1.3 (772)",
         "TLS 1.2 (771)",
         "TLS 1.1 (770)",
@@ -127,7 +127,7 @@
     },
     {
       "type": "GREASE",
-      "decimal": 39578,
+      "decimal": 6682,
       "values": ["0000 - 00                                             ."]
     },
     {

diff --git a/...s/chrome-80/data/mac-os-10-10/codecs.json → ...hrome-80/data/as-mac-os-10-10/codecs.json b/...s/chrome-80/data/mac-os-10-10/codecs.json → ...hrome-80/data/as-mac-os-10-10/codecs.json