feat(telemetry): adding yarn 1 support, sanitizing data pre-flight (#…

…3082)

add support for yarn 1 in telemetry reporting

sanitizes two data points pre-flight - the system version (Node) and the 
`packages` array

package.json

@@ -79,6 +79,8 @@
     "@types/sizzle": "^2.3.2",
     "@types/webpack": "^4.41.26",
     "@types/ws": "^7.4.0",
+    "@types/yarnpkg__lockfile": "^1.1.5",
+    "@yarnpkg/lockfile": "^1.1.0",
     "ansi-colors": "4.1.1",
     "autoprefixer": "10.2.5",
     "concurrently": "^6.0.0",

scripts/license.ts

@@ -10,6 +10,7 @@ const entryDeps = [
   '@rollup/plugin-commonjs',
   '@rollup/plugin-json',
   '@rollup/plugin-node-resolve',
+  '@yarnpkg/lockfile',
   'ansi-colors',
   'autoprefixer',
   'css',

src/cli/telemetry/telemetry.ts

@@ -107,7 +107,7 @@ export const prepareData = async (
   component_count: number = undefined
 ): Promise<d.TrackableData> => {
   const { typescript, rollup } = coreCompiler.versions || { typescript: 'unknown', rollup: 'unknown' };
-  const { packages } = await getInstalledPackages(sys, config);
+  const { packages, packagesNoVersions } = await getInstalledPackages(sys, config);
   const targets = await getActiveTargets(config);
   const yarn = isUsingYarn(sys);
   const stencil = coreCompiler.version || 'unknown';
@@ -124,10 +124,12 @@ export const prepareData = async (
     component_count,
     targets,
     packages,
+    packages_no_versions: packagesNoVersions,
     arguments: config.flags.args,
     task: config.flags.task,
     stencil,
     system,
+    system_major: getMajorVersion(system),
     os_name,
     os_version,
     cpu_model,
@@ -139,24 +141,28 @@ export const prepareData = async (
 };
 
 /**
- * Reads package-lock.json and package.json files in order to cross references the dependencies and devDependencies properties. Pull the current installed version of each package under the @stencil, @ionic, and @capacitor scopes.
+ * Reads package-lock.json, yarn.lock, and package.json files in order to cross reference
+ * the dependencies and devDependencies properties. Pulls up the current installed version
+ * of each package under the @stencil, @ionic, and @capacitor scopes.
  * @returns string[]
  */
-async function getInstalledPackages(sys: d.CompilerSystem, config: d.Config): Promise<{ packages: string[] }> {
+async function getInstalledPackages(
+  sys: d.CompilerSystem,
+  config: d.Config
+): Promise<{ packages: string[]; packagesNoVersions: string[] }> {
   let packages: string[] = [];
-  let packageLockJson: any;
+  let packagesNoVersions: string[] = [];
+  const yarn = isUsingYarn(sys);
 
   try {
     // Read package.json and package-lock.json
     const appRootDir = sys.getCurrentDirectory();
 
-    const packageJson: d.PackageJsonData = await tryFn(readJson, sys.resolvePath(appRootDir + '/package.json'));
-
-    packageLockJson = await tryFn(readJson, sys.resolvePath(appRootDir + '/package-lock.json'));
+    const packageJson: d.PackageJsonData = await tryFn(readJson, sys, sys.resolvePath(appRootDir + '/package.json'));
 
     // They don't have a package.json for some reason? Eject button.
     if (!packageJson) {
-      return { packages };
+      return { packages, packagesNoVersions };
     }
 
     const rawPackages: [string, string][] = Object.entries({
@@ -170,20 +176,67 @@ async function getInstalledPackages(sys: d.CompilerSystem, config: d.Config): Pr
       ([k]) => k.startsWith('@stencil/') || k.startsWith('@ionic/') || k.startsWith('@capacitor/')
     );
 
-    packages = packageLockJson
-      ? ionicPackages.map(
-          ([k, v]) =>
-            `${k}@${packageLockJson?.dependencies[k]?.version ?? packageLockJson?.devDependencies[k]?.version ?? v}`
-        )
-      : ionicPackages.map(([k, v]) => `${k}@${v}`);
+    try {
+      packages = yarn ? await yarnPackages(sys, ionicPackages) : await npmPackages(sys, ionicPackages);
+    } catch (e) {
+      packages = ionicPackages.map(([k, v]) => `${k}@${v.replace('^', '')}`);
+    }
+
+    packagesNoVersions = ionicPackages.map(([k]) => `${k}`);
 
-    return { packages };
+    return { packages, packagesNoVersions };
   } catch (err) {
     hasDebug(config) && console.error(err);
-    return { packages };
+    return { packages, packagesNoVersions };
   }
 }
 
+/**
+ * Visits the npm lock file to find the exact versions that are installed
+ * @param sys The system where the command is invoked
+ * @param ionicPackages a list of the found packages matching `@stencil`, `@capacitor`, or `@ionic` from the package.json file.
+ * @returns an array of strings of all the packages and their versions.
+ */
+async function npmPackages(sys: d.CompilerSystem, ionicPackages: [string, string][]): Promise<string[]> {
+  const appRootDir = sys.getCurrentDirectory();
+  const packageLockJson: any = await tryFn(readJson, sys, sys.resolvePath(appRootDir + '/package-lock.json'));
+
+  return ionicPackages.map(([k, v]) => {
+    let version = packageLockJson?.dependencies[k]?.version ?? packageLockJson?.devDependencies[k]?.version ?? v;
+    version = version.includes('file:') ? sanitizeDeclaredVersion(v) : version;
+    return `${k}@${version}`;
+  });
+}
+
+/**
+ * Visits the yarn lock file to find the exact versions that are installed
+ * @param sys The system where the command is invoked
+ * @param ionicPackages a list of the found packages matching `@stencil`, `@capacitor`, or `@ionic` from the package.json file.
+ * @returns an array of strings of all the packages and their versions.
+ */
+async function yarnPackages(sys: d.CompilerSystem, ionicPackages: [string, string][]): Promise<string[]> {
+  const appRootDir = sys.getCurrentDirectory();
+  const yarnLock = sys.readFileSync(sys.resolvePath(appRootDir + '/yarn.lock'));
+  const yarnLockYml = sys.parseYarnLockFile(yarnLock);
+
+  return ionicPackages.map(([k, v]) => {
+    const identifiedVersion = `${k}@${v}`;
+    let version = yarnLockYml.object[identifiedVersion]?.version;
+    version = version.includes('undefined') ? sanitizeDeclaredVersion(identifiedVersion) : version;
+    return `${k}@${version}`;
+  });
+}
+
+/**
+ * This function is used for fallback purposes, where an npm or yarn lock file doesn't exist in the consumers directory.
+ * This will strip away '*', '^' and '~' from the declared package versions in a package.json.
+ * @param version the raw semver pattern identifier version string
+ * @returns a cleaned up representation without any qualifiers
+ */
+function sanitizeDeclaredVersion(version: string): string {
+  return version.replace(/[*^~]/g, '');
+}
+
 /**
  * If telemetry is enabled, send a metric via IPC to a forked process for uploading.
  */
@@ -287,3 +340,13 @@ export async function enableTelemetry(sys: d.CompilerSystem): Promise<boolean> {
 export async function disableTelemetry(sys: d.CompilerSystem): Promise<boolean> {
   return await updateConfig(sys, { 'telemetry.stencil': false });
 }
+
+/**
+ * Takes in a semver string in order to return the major version.
+ * @param version The fully qualified semver version
+ * @returns a string of the major version
+ */
+function getMajorVersion(version: string): string {
+  const parts = version.split('.');
+  return parts[0];
+}

src/cli/telemetry/test/telemetry.spec.ts

@@ -148,9 +148,11 @@ describe('prepareData', () => {
       os_name: '',
       os_version: '',
       packages: [],
+      packages_no_versions: [],
       rollup: 'unknown',
       stencil: 'unknown',
       system: 'in-memory __VERSION:STENCIL__',
+      system_major: 'in-memory __VERSION:STENCIL__',
       targets: [],
       task: undefined,
       typescript: 'unknown',
@@ -178,9 +180,11 @@ describe('prepareData', () => {
       os_name: '',
       os_version: '',
       packages: [],
+      packages_no_versions: [],
       rollup: 'unknown',
       stencil: 'unknown',
       system: 'in-memory __VERSION:STENCIL__',
+      system_major: 'in-memory __VERSION:STENCIL__',
       targets: ['www'],
       task: undefined,
       typescript: 'unknown',
@@ -208,9 +212,11 @@ describe('prepareData', () => {
       os_name: '',
       os_version: '',
       packages: [],
+      packages_no_versions: [],
       rollup: 'unknown',
       stencil: 'unknown',
       system: 'in-memory __VERSION:STENCIL__',
+      system_major: 'in-memory __VERSION:STENCIL__',
       targets: ['www'],
       task: undefined,
       typescript: 'unknown',

src/declarations/stencil-public-compiler.ts

@@ -998,6 +998,10 @@ export interface CompilerSystem {
    */
   normalizePath(p: string): string;
   onProcessInterrupt?(cb: () => void): void;
+  parseYarnLockFile?: (content: string) => {
+    type: 'success' | 'merge' | 'conflict';
+    object: any;
+  };
   platformPath: PlatformPath;
   /**
    * All return paths are full normalized paths, not just the basenames. Always returns an array, does not throw.

src/sys/node/node-sys.ts

@@ -12,6 +12,7 @@ import { cpus, freemem, platform, release, tmpdir, totalmem } from 'os';
 import { createHash } from 'crypto';
 import exit from 'exit';
 import fs from 'graceful-fs';
+import { parse as parseYarnLockFile } from '@yarnpkg/lockfile';
 import { isFunction, isPromise, normalizePath } from '@utils';
 import { NodeLazyRequire } from './node-lazy-require';
 import { NodeResolveModule } from './node-resolve-module';
@@ -256,6 +257,9 @@ export function createNodeSys(c: { process?: any } = {}) {
         });
       });
     },
+    parseYarnLockFile(content: string) {
+      return parseYarnLockFile(content);
+    },
     isTTY() {
       return !!process?.stdout?.isTTY;
     },