[labs/analyzer] More analysis coverage for gen-manifest (#3529)

* Print diagnostic errors more gracefully to CLI.

* Add support for --exclude options (e.g. exclude tests)

* Add support for TS enum type variables.

* Parse & emit description, summary, and deprecated for all models.

* Add support for module-level descriptions.

* Add class member analysis.

* Fix jsdoc tests.

* Add tests

* Update goldens

* Add changeset

* Revert error logging change

* Normalize JSDoc parsing for events, fixes Windows issue

* Print diagnostic errors more gracefully to CLI

* Line ending fixes for Windows

* Add tests for module-level JSDoc, and improve logic & comments

* Remove heuristic for parsing @summary

* Fix @slot to require dash before comment
Also fixes accidentally skipped tests re: summary and removes more unused code.

* Fix typo in regex

* Be more lenient to casing on jsDoc tags.

* Update test project JSDoc to match updates to analyzer rules

* Address review feedback

* Apply suggestions from code review

Co-authored-by: Justin Fagnani <justinfagnani@google.com>

* Refactor to take advantage of undefined not JSON.serialize'd

* Improve naming of model heritage

Co-authored-by: Justin Fagnani <justinfagnani@google.com>

.changeset/stupid-shrimps-promise.md

@@ -0,0 +1,16 @@
+---
+'@lit-labs/analyzer': minor
+'@lit-labs/cli': minor
+'@lit-labs/gen-manifest': minor
+---
+
+Added CLI improvements:
+
+- Add support for --exclude options (important for excluding test files from e.g. manifest or wrapper generation)
+
+Added more analysis support and manifest emit:
+
+- TS enum type variables
+- description, summary, and deprecated for all models
+- module-level description & summary
+- ClassField and ClassMethod

packages/labs/analyzer/.vscode/launch.json

@@ -11,6 +11,7 @@
       "skipFiles": ["<node_internals>/**"],
       "program": "${workspaceFolder}/../../../node_modules/uvu/bin.js",
       "args": ["test", "\\_test\\.js$"],
+      "env": {"NODE_OPTIONS": "--enable-source-maps"},
       "outFiles": ["${workspaceFolder}/**/*.js"],
       "console": "integratedTerminal"
     }

packages/labs/analyzer/src/index.ts

@@ -16,10 +16,15 @@ export type {
   Declaration,
   VariableDeclaration,
   ClassDeclaration,
+  ClassField,
+  ClassMethod,
+  Parameter,
+  Return,
   LitElementDeclaration,
   LitElementExport,
   PackageJson,
   ModuleWithLitElementDeclarations,
+  DeprecatableDescribed,
 } from './lib/model.js';
 
 export type {AbsolutePath, PackagePath} from './lib/paths.js';

packages/labs/analyzer/src/lib/analyze-package.ts

@@ -1,17 +1,36 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
 import ts from 'typescript';
 import {AbsolutePath} from './paths.js';
 import * as path from 'path';
 import {DiagnosticsError} from './errors.js';
 import {Analyzer} from './analyzer.js';
 
+export interface AnalyzerOptions {
+  /**
+   * Glob of source files to exclude from project during analysis.
+   *
+   * Useful for excluding things source like test folders that might otherwise
+   * be included in a project's tsconfig.
+   */
+  exclude?: string[];
+}
+
 /**
  * Returns an analyzer for a Lit npm package based on a filesystem path.
  *
  * The path may specify a package root folder, or a specific tsconfig file. When
  * specifying a folder, if no tsconfig.json file is found directly in the root
  * folder, the project will be analyzed as JavaScript.
  */
-export const createPackageAnalyzer = (packagePath: AbsolutePath) => {
+export const createPackageAnalyzer = (
+  packagePath: AbsolutePath,
+  options: AnalyzerOptions = {}
+) => {
   // This logic accepts either a path to folder containing a tsconfig.json
   // directly inside it or a path to a specific tsconfig file. If no tsconfig
   // file is found, we fallback to creating a Javascript program.
@@ -22,6 +41,9 @@ export const createPackageAnalyzer = (packagePath: AbsolutePath) => {
   let commandLine: ts.ParsedCommandLine;
   if (ts.sys.fileExists(configFileName)) {
     const configFile = ts.readConfigFile(configFileName, ts.sys.readFile);
+    if (options.exclude !== undefined) {
+      (configFile.config.exclude ??= []).push(...options.exclude);
+    }
     commandLine = ts.parseJsonConfigFileContent(
       configFile.config /* json */,
       ts.sys /* host */,
@@ -55,6 +77,7 @@ export const createPackageAnalyzer = (packagePath: AbsolutePath) => {
           moduleResolution: 'node',
         },
         include: ['**/*.js'],
+        exclude: options.exclude ?? [],
       },
       ts.sys /* host */,
       packagePath /* basePath */

packages/labs/analyzer/src/lib/javascript/classes.ts

@@ -18,12 +18,23 @@ import {
   DeclarationInfo,
   ClassHeritage,
   Reference,
+  ClassField,
+  ClassMethod,
 } from '../model.js';
 import {
   isLitElementSubclass,
   getLitElementDeclaration,
 } from '../lit-element/lit-element.js';
-import {hasExportKeyword, getReferenceForIdentifier} from '../references.js';
+import {getReferenceForIdentifier} from '../references.js';
+import {parseNodeJSDocInfo} from './jsdoc.js';
+import {
+  hasDefaultModifier,
+  hasStaticModifier,
+  hasExportModifier,
+  getPrivacy,
+} from '../utils.js';
+import {getFunctionLikeInfo} from './functions.js';
+import {getTypeForNode} from '../types.js';
 
 /**
  * Returns an analyzer `ClassDeclaration` model for the given
@@ -41,7 +52,54 @@ const getClassDeclaration = (
     name: declaration.name?.text ?? '',
     node: declaration,
     getHeritage: () => getHeritage(declaration, analyzer),
+    ...parseNodeJSDocInfo(declaration),
+    ...getClassMembers(declaration, analyzer),
+  });
+};
+
+/**
+ * Returns the `fields` and `methods` of a class.
+ */
+export const getClassMembers = (
+  declaration: ts.ClassDeclaration,
+  analyzer: AnalyzerInterface
+) => {
+  const fieldMap = new Map<string, ClassField>();
+  const methodMap = new Map<string, ClassMethod>();
+  declaration.members.forEach((node) => {
+    if (ts.isMethodDeclaration(node)) {
+      methodMap.set(
+        node.name.getText(),
+        new ClassMethod({
+          ...getMemberInfo(node),
+          ...getFunctionLikeInfo(node, analyzer),
+          ...parseNodeJSDocInfo(node),
+        })
+      );
+    } else if (ts.isPropertyDeclaration(node)) {
+      fieldMap.set(
+        node.name.getText(),
+        new ClassField({
+          ...getMemberInfo(node),
+          default: node.initializer?.getText(),
+          type: getTypeForNode(node, analyzer),
+          ...parseNodeJSDocInfo(node),
+        })
+      );
+    }
   });
+  return {
+    fieldMap,
+    methodMap,
+  };
+};
+
+const getMemberInfo = (node: ts.MethodDeclaration | ts.PropertyDeclaration) => {
+  return {
+    name: node.name.getText(),
+    static: hasStaticModifier(node),
+    privacy: getPrivacy(node),
+  };
 };
 
 /**
@@ -52,9 +110,7 @@ const getClassDeclarationName = (declaration: ts.ClassDeclaration) => {
     declaration.name?.text ??
     // The only time a class declaration will not have a name is when it is
     // a default export, aka `export default class { }`
-    (declaration.modifiers?.some((s) => s.kind === ts.SyntaxKind.DefaultKeyword)
-      ? 'default'
-      : undefined);
+    (hasDefaultModifier(declaration) ? 'default' : undefined);
   if (name === undefined) {
     throw new DiagnosticsError(
       declaration,
@@ -74,7 +130,7 @@ export const getClassDeclarationInfo = (
   return {
     name: getClassDeclarationName(declaration),
     factory: () => getClassDeclaration(declaration, analyzer),
-    isExport: hasExportKeyword(declaration),
+    isExport: hasExportModifier(declaration),
   };
 };
 

packages/labs/analyzer/src/lib/javascript/functions.ts

@@ -0,0 +1,81 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @fileoverview
+ *
+ * Utilities for analyzing function declarations
+ */
+
+import ts from 'typescript';
+import {DiagnosticsError} from '../errors.js';
+import {AnalyzerInterface, Parameter, Return} from '../model.js';
+import {getTypeForNode, getTypeForType} from '../types.js';
+import {parseJSDocDescription} from './jsdoc.js';
+
+/**
+ * Returns information on FunctionLike nodes
+ */
+export const getFunctionLikeInfo = (
+  node: ts.FunctionLikeDeclaration,
+  analyzer: AnalyzerInterface
+) => {
+  return {
+    parameters: node.parameters.map((p) => getParameter(p, analyzer)),
+    return: getReturn(node, analyzer),
+  };
+};
+
+const getParameter = (
+  param: ts.ParameterDeclaration,
+  analyzer: AnalyzerInterface
+): Parameter => {
+  const paramTag = ts.getAllJSDocTagsOfKind(
+    param,
+    ts.SyntaxKind.JSDocParameterTag
+  )[0];
+  const p: Parameter = {
+    name: param.name.getText(),
+    type: getTypeForNode(param, analyzer),
+    ...(paramTag ? parseJSDocDescription(paramTag) : {}),
+    optional: false,
+    rest: false,
+  };
+  if (param.initializer !== undefined) {
+    p.optional = true;
+    p.default = param.initializer.getText();
+  }
+  if (param.questionToken !== undefined) {
+    p.optional = true;
+  }
+  if (param.dotDotDotToken !== undefined) {
+    p.rest = true;
+  }
+  return p;
+};
+
+const getReturn = (
+  node: ts.FunctionLikeDeclaration,
+  analyzer: AnalyzerInterface
+): Return => {
+  const returnTag = ts.getAllJSDocTagsOfKind(
+    node,
+    ts.SyntaxKind.JSDocReturnTag
+  )[0];
+  const signature = analyzer.program
+    .getTypeChecker()
+    .getSignatureFromDeclaration(node);
+  if (signature === undefined) {
+    throw new DiagnosticsError(
+      node,
+      `Could not get signature to determine return type`
+    );
+  }
+  return {
+    type: getTypeForType(signature.getReturnType(), node, analyzer),
+    ...(returnTag ? parseJSDocDescription(returnTag) : {}),
+  };
+};