perf(vscode): Cache node_modules files lazily (#891)

nrwl · Nov 20, 2019 · 336ee76 · 336ee76
1 parent 919ca16
commit 336ee76
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 99 deletions.
diff --git a/apps/vscode/src/main.ts b/apps/vscode/src/main.ts
@@ -1,4 +1,4 @@
-import { cacheJsonFiles, EXTENSIONS } from '@angular-console/server';
+import { EXTENSIONS } from '@angular-console/server';
 import { stream } from 'fast-glob';
 import { existsSync } from 'fs';
 import { dirname, join, parse } from 'path';
@@ -204,35 +204,8 @@ async function setAngularWorkspace(workspacePath: string) {
     ngTaskProvider.setWorkspacePath(workspacePath);
   }
 
-  setTimeout(() => {
-    cacheWorkspaceNodeModulesJsons(workspacePath);
-  }, 0);
-  setInterval(() => cacheWorkspaceNodeModulesJsons(workspacePath), 60000);
-
   commands.executeCommand('setContext', 'isAngularWorkspace', true);
 
   currentWorkspaceTreeProvider.setWorkspacePath(workspacePath);
   angularJsonTreeProvider.setWorkspacePath(workspacePath);
 }
-
-function cacheWorkspaceNodeModulesJsons(workspacePath: string) {
-  if (!existsSync(join(workspacePath, 'node_modules'))) {
-    getOutputChannel().appendLine(
-      'Tried to cache node_modules but directory was not present. Run npm install'
-    );
-    return;
-  }
-
-  try {
-    cacheJsonFiles(workspacePath);
-  } catch (e) {
-    window.showErrorMessage(
-      'Angular Console encountered an error when scanning node_modules'
-    );
-    getOutputChannel().appendLine('Error parsing node_modules ');
-
-    const stringifiedError = e.toString ? e.toString() : JSON.stringify(e);
-    getOutputChannel().appendLine(stringifiedError);
-    getTelemetry().exceptionOccured(stringifiedError);
-  }
-}
diff --git a/libs/server/src/index.ts b/libs/server/src/index.ts
@@ -2,9 +2,7 @@ export { readAndParseJson } from './lib/utils/utils';
 
 export { findClosestNg } from './lib/utils/utils';
 
-export { cacheJsonFiles } from './lib/utils/utils';
-
-export { readJsonFile } from './lib/utils/utils';
+export { readAndCacheJsonFile as readJsonFile } from './lib/utils/utils';
 
 export { Telemetry } from './lib/telemetry';
 export * from './lib/extensions';

diff --git a/libs/server/src/lib/read-schematic-collections.ts b/libs/server/src/lib/read-schematic-collections.ts
@@ -7,7 +7,7 @@ import {
   listFiles,
   listOfUnnestedNpmPackages,
   normalizeSchema,
-  readJsonFile
+  readAndCacheJsonFile
 } from './utils/utils';
 
 export function readAllSchematicCollections(
@@ -38,7 +38,8 @@ export function readAllSchematicCollections(
 }
 
 function readAngularJsonDefaults(basedir: string): any {
-  const defaults = readJsonFile('angular.json', basedir).json.schematics;
+  const defaults = readAndCacheJsonFile('angular.json', basedir).json
+    .schematics;
   const collectionDefaults = Object.keys(defaults ? defaults : {}).reduce(
     (collectionDefaultsMap: any, key) => {
       const [collectionName, schematicName] = key.split(':');
@@ -60,8 +61,10 @@ function readSchematicCollectionsFromNodeModules(
   const packages = listOfUnnestedNpmPackages(nodeModulesDir);
   const schematicCollections = packages.filter(p => {
     try {
-      return !!readJsonFile(path.join(p, 'package.json'), nodeModulesDir).json
-        .schematics;
+      return !!readAndCacheJsonFile(
+        path.join(p, 'package.json'),
+        nodeModulesDir
+      ).json.schematics;
     } catch (e) {
       if (
         e.message &&
@@ -90,7 +93,7 @@ function readWorkspaceSchematicsCollection(
   const collectionDir = path.join(basedir, workspaceSchematicsPath);
   const collectionName = '@nrwl/workspace';
   if (fileExistsSync(path.join(collectionDir, 'collection.json'))) {
-    const collection = readJsonFile('collection.json', collectionDir);
+    const collection = readAndCacheJsonFile('collection.json', collectionDir);
     const defaults = readAngularJsonDefaults(basedir);
 
     return readCollectionSchematics(
@@ -103,7 +106,7 @@ function readWorkspaceSchematicsCollection(
     const schematics = listFiles(collectionDir)
       .filter(f => path.basename(f) === 'schema.json')
       .map(f => {
-        const schemaJson = readJsonFile(f, '');
+        const schemaJson = readAndCacheJsonFile(f, '');
         return {
           name: schemaJson.json.id,
           collection: collectionName,
@@ -123,11 +126,11 @@ function readCollection(
   defaults?: any
 ): SchematicCollection | null {
   try {
-    const packageJson = readJsonFile(
+    const packageJson = readAndCacheJsonFile(
       path.join(collectionName, 'package.json'),
       basedir
     );
-    const collection = readJsonFile(
+    const collection = readAndCacheJsonFile(
       packageJson.json.schematics,
       path.dirname(packageJson.path)
     );
@@ -156,7 +159,7 @@ function readCollectionSchematics(
   Object.entries(collectionJson.schematics).forEach(([k, v]: [any, any]) => {
     try {
       if (canAdd(k, v)) {
-        const schematicSchema = readJsonFile(
+        const schematicSchema = readAndCacheJsonFile(
           v.schema,
           path.dirname(collectionPath)
         );
@@ -167,9 +170,7 @@ function readCollectionSchematics(
           name: k,
           collection: collectionName,
           schema: normalizeSchema(schematicSchema.json, projectDefaults),
-          description: v.description || '',
-          npmClient: null,
-          npmScript: null
+          description: v.description || ''
         });
       }
     } catch (e) {

diff --git a/libs/server/src/lib/utils/read-projects.ts b/libs/server/src/lib/utils/read-projects.ts
@@ -4,7 +4,7 @@ import * as path from 'path';
 import {
   getPrimitiveValue,
   normalizeSchema,
-  readJsonFile
+  readAndCacheJsonFile
 } from '../utils/utils';
 
 export function readProjects(json: any): Project[] {
@@ -83,20 +83,20 @@ export function readSchema(basedir: string, builder: string) {
 }
 
 function readBuildersFile(basedir: string, npmPackage: string): any {
-  const packageJson = readJsonFile(
+  const packageJson = readAndCacheJsonFile(
     path.join(npmPackage, 'package.json'),
     path.join(basedir, 'node_modules')
   );
   const b = packageJson.json.builders;
   const buildersPath = b.startsWith('.') ? b : `./${b}`;
-  const buildersJson = readJsonFile(
+  const buildersJson = readAndCacheJsonFile(
     buildersPath,
     path.dirname(packageJson.path)
   );
 
   const builders = {} as any;
   Object.entries(buildersJson.json.builders).forEach(([k, v]: [any, any]) => {
-    const builderSchema = readJsonFile(
+    const builderSchema = readAndCacheJsonFile(
       v.schema,
       path.dirname(buildersJson.path)
     );

diff --git a/libs/server/src/lib/utils/read-scehmatic-collections.ts b/libs/server/src/lib/utils/read-scehmatic-collections.ts
@@ -3,7 +3,7 @@ import * as path from 'path';
 import {
   fileExistsSync,
   directoryExists,
-  readJsonFile,
+  readAndCacheJsonFile,
   listOfUnnestedNpmPackages,
   normalizeSchema,
   listFiles
@@ -37,7 +37,8 @@ export function readAllSchematicCollections(
 }
 
 function readAngularJsonDefaults(basedir: string): any {
-  const defaults = readJsonFile('angular.json', basedir).json.schematics;
+  const defaults = readAndCacheJsonFile('angular.json', basedir).json
+    .schematics;
   const collectionDefaults = Object.keys(defaults ? defaults : {}).reduce(
     (collectionDefaultsMap: any, key) => {
       const [collectionName, schematicName] = key.split(':');
@@ -59,8 +60,10 @@ function readSchematicCollectionsFromNodeModules(
   const packages = listOfUnnestedNpmPackages(nodeModulesDir);
   const schematicCollections = packages.filter(p => {
     try {
-      return !!readJsonFile(path.join(p, 'package.json'), nodeModulesDir).json
-        .schematics;
+      return !!readAndCacheJsonFile(
+        path.join(p, 'package.json'),
+        nodeModulesDir
+      ).json.schematics;
     } catch (e) {
       if (
         e.message &&
@@ -89,7 +92,7 @@ function readWorkspaceSchematicsCollection(
   const collectionDir = path.join(basedir, workspaceSchematicsPath);
   const collectionName = '@nrwl/workspace';
   if (fileExistsSync(path.join(collectionDir, 'collection.json'))) {
-    const collection = readJsonFile('collection.json', collectionDir);
+    const collection = readAndCacheJsonFile('collection.json', collectionDir);
     const defaults = readAngularJsonDefaults(basedir);
 
     return readCollectionSchematics(
@@ -102,7 +105,7 @@ function readWorkspaceSchematicsCollection(
     const schematics = listFiles(collectionDir)
       .filter(f => path.basename(f) === 'schema.json')
       .map(f => {
-        const schemaJson = readJsonFile(f, '');
+        const schemaJson = readAndCacheJsonFile(f, '');
         return {
           name: schemaJson.json.id,
           collection: collectionName,
@@ -122,11 +125,11 @@ function readCollection(
   defaults?: any
 ): SchematicCollection | null {
   try {
-    const packageJson = readJsonFile(
+    const packageJson = readAndCacheJsonFile(
       path.join(collectionName, 'package.json'),
       basedir
     );
-    const collection = readJsonFile(
+    const collection = readAndCacheJsonFile(
       packageJson.json.schematics,
       path.dirname(packageJson.path)
     );
@@ -155,7 +158,7 @@ function readCollectionSchematics(
   Object.entries(collectionJson.schematics).forEach(([k, v]: [any, any]) => {
     try {
       if (canAdd(k, v)) {
-        const schematicSchema = readJsonFile(
+        const schematicSchema = readAndCacheJsonFile(
           v.schema,
           path.dirname(collectionPath)
         );

diff --git a/libs/server/src/lib/utils/utils.ts b/libs/server/src/lib/utils/utils.ts
@@ -127,21 +127,6 @@ export function listFiles(dirName: string): string[] {
   return res;
 }
 
-export function cacheJsonFiles(basedir: string) {
-  const nodeModulesDir = path.join(basedir, 'node_modules');
-  const packages = listOfUnnestedNpmPackages(nodeModulesDir);
-
-  const res: any = {};
-  packages.forEach(p => {
-    const filePath = path.join(nodeModulesDir, p, 'package.json');
-    if (!fileExistsSync(filePath)) return;
-    res[filePath] = readAndParseJson(
-      path.join(nodeModulesDir, p, 'package.json')
-    );
-  });
-  return res;
-}
-
 export function directoryExists(filePath: string): boolean {
   try {
     return statSync(filePath).isDirectory();
@@ -162,25 +147,20 @@ export function readAndParseJson(fullFilePath: string): any {
   return JSON.parse(stripJsonComments(readFileSync(fullFilePath).toString()));
 }
 
-export function readJsonFile(
+export function readAndCacheJsonFile(
   filePath: string,
   basedir: string
 ): { path: string; json: any } {
   const fullFilePath = path.join(basedir, filePath);
 
-  // we can try to retrieve node_modules files from the cache because
-  // they don't change very often
-  const cache = basedir.endsWith('node_modules');
-  if (cache && fileContents[fullFilePath]) {
+  if (fileContents[fullFilePath] || existsSync(fullFilePath)) {
+    fileContents[fullFilePath] =
+      fileContents[fullFilePath] || readAndParseJson(fullFilePath);
+
     return {
       path: fullFilePath,
       json: fileContents[fullFilePath]
     };
-  } else if (existsSync(fullFilePath)) {
-    return {
-      path: fullFilePath,
-      json: readAndParseJson(fullFilePath)
-    };
   } else {
     return {
       path: fullFilePath,
@@ -309,21 +289,3 @@ export function seconds<T>(fn: Function): [number, T] {
   const end = process.hrtime(start);
   return [end[0], result];
 }
-
-/**
- * To improve performance angular console pre-processes
- *
- * * the list of local files
- * * json files from node_modules we are likely to read
- *
- * both the data sets get updated every 60 seconds.
- */
-export function cacheFiles(p: string) {
-  setTimeout(() => {
-    files[p] = listFiles(p);
-    fileContents = cacheJsonFiles(p);
-    setTimeout(() => {
-      cacheFiles(p);
-    }, 60000);
-  }, 0);
-}