fix(rosetta): types from submodules not recognized properly (#3174)

Rosetta used to store a guessed version of the type fqn when running `rosetta extract`. These guessed fqns are correct for v1 but break down on v2, since they do not properly account for namespaces. This PR correctly determines the fqn of a type by computing the symbolId, loading the relevant assembly, and matching the symbolId with the actual fqn.

---

By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license].

[Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0

packages/@scope/jsii-calc-lib/test/assembly.jsii

@@ -811,7 +811,7 @@
           }
         }
       ],
-      "symbolId": "lib/submodule/index:NestedClass"
+      "symbolId": "lib/submodule/index:NestingClass.NestedClass"
     },
     "@scope/jsii-calc-lib.submodule.NestingClass.NestedStruct": {
       "assembly": "@scope/jsii-calc-lib",
@@ -846,7 +846,7 @@
           }
         }
       ],
-      "symbolId": "lib/submodule/index:NestedStruct"
+      "symbolId": "lib/submodule/index:NestingClass.NestedStruct"
     },
     "@scope/jsii-calc-lib.submodule.ReflectableEntry": {
       "assembly": "@scope/jsii-calc-lib",
@@ -948,5 +948,5 @@
     }
   },
   "version": "0.0.0",
-  "fingerprint": "BXEo4aMVmNYUV0dd8hK2zOYLM8iFbpxQbUzGyGagFu8="
+  "fingerprint": "LN1bs46m2O4aR6AhHvi6UDh/f90EoUT3n5apMPzr9+c="
 }

packages/jsii-calc/test/assembly.jsii

@@ -4082,7 +4082,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:Base"
+      "symbolId": "lib/compliance:DerivedClassHasNoProperties.Base"
     },
     "jsii-calc.DerivedClassHasNoProperties.Derived": {
       "assembly": "jsii-calc",
@@ -4103,7 +4103,7 @@
       },
       "name": "Derived",
       "namespace": "DerivedClassHasNoProperties",
-      "symbolId": "lib/compliance:Derived"
+      "symbolId": "lib/compliance:DerivedClassHasNoProperties.Derived"
     },
     "jsii-calc.DerivedStruct": {
       "assembly": "jsii-calc",
@@ -7562,7 +7562,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:Foo"
+      "symbolId": "lib/compliance:InterfaceInNamespaceIncludesClasses.Foo"
     },
     "jsii-calc.InterfaceInNamespaceIncludesClasses.Hello": {
       "assembly": "jsii-calc",
@@ -7595,7 +7595,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:Hello"
+      "symbolId": "lib/compliance:InterfaceInNamespaceIncludesClasses.Hello"
     },
     "jsii-calc.InterfaceInNamespaceOnlyInterface.Hello": {
       "assembly": "jsii-calc",
@@ -7628,7 +7628,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:Hello"
+      "symbolId": "lib/compliance:InterfaceInNamespaceOnlyInterface.Hello"
     },
     "jsii-calc.InterfacesMaker": {
       "assembly": "jsii-calc",
@@ -9004,7 +9004,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:PropBooleanValue"
+      "symbolId": "lib/compliance:LevelOne.PropBooleanValue"
     },
     "jsii-calc.LevelOne.PropProperty": {
       "assembly": "jsii-calc",
@@ -9037,7 +9037,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:PropProperty"
+      "symbolId": "lib/compliance:LevelOne.PropProperty"
     },
     "jsii-calc.LevelOneProps": {
       "assembly": "jsii-calc",
@@ -11141,7 +11141,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:ClassWithSelf"
+      "symbolId": "lib/compliance:PythonSelf.ClassWithSelf"
     },
     "jsii-calc.PythonSelf.ClassWithSelfKwarg": {
       "assembly": "jsii-calc",
@@ -11189,7 +11189,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:ClassWithSelfKwarg"
+      "symbolId": "lib/compliance:PythonSelf.ClassWithSelfKwarg"
     },
     "jsii-calc.PythonSelf.IInterfaceWithSelf": {
       "assembly": "jsii-calc",
@@ -11230,7 +11230,7 @@
       ],
       "name": "IInterfaceWithSelf",
       "namespace": "PythonSelf",
-      "symbolId": "lib/compliance:IInterfaceWithSelf"
+      "symbolId": "lib/compliance:PythonSelf.IInterfaceWithSelf"
     },
     "jsii-calc.PythonSelf.StructWithSelf": {
       "assembly": "jsii-calc",
@@ -11263,7 +11263,7 @@
           }
         }
       ],
-      "symbolId": "lib/compliance:StructWithSelf"
+      "symbolId": "lib/compliance:PythonSelf.StructWithSelf"
     },
     "jsii-calc.ReferenceEnumFromScopedPackage": {
       "assembly": "jsii-calc",
@@ -14863,7 +14863,7 @@
           }
         }
       ],
-      "symbolId": "lib/calculator:CompositeOperation"
+      "symbolId": "lib/calculator:composition.CompositeOperation"
     },
     "jsii-calc.composition.CompositeOperation.CompositionStringStyle": {
       "assembly": "jsii-calc",
@@ -14895,7 +14895,7 @@
       ],
       "name": "CompositionStringStyle",
       "namespace": "composition.CompositeOperation",
-      "symbolId": "lib/calculator:CompositionStringStyle"
+      "symbolId": "lib/calculator:composition.CompositeOperation.CompositionStringStyle"
     },
     "jsii-calc.module2530.MyClass": {
       "assembly": "jsii-calc",
@@ -16673,7 +16673,7 @@
           }
         }
       ],
-      "symbolId": "lib/submodule/nested_submodule:Namespaced"
+      "symbolId": "lib/submodule/nested_submodule:nested_submodule.Namespaced"
     },
     "jsii-calc.submodule.nested_submodule.deeplyNested.INamespaced": {
       "assembly": "jsii-calc",
@@ -16705,7 +16705,7 @@
           }
         }
       ],
-      "symbolId": "lib/submodule/nested_submodule:INamespaced"
+      "symbolId": "lib/submodule/nested_submodule:nested_submodule.deeplyNested.INamespaced"
     },
     "jsii-calc.submodule.param.SpecialParameter": {
       "assembly": "jsii-calc",
@@ -16779,5 +16779,5 @@
     }
   },
   "version": "3.20.120",
-  "fingerprint": "2ST30e+9gb6XYfIMHyGjwObar3fI+/BV0Ex1hIad+2s="
+  "fingerprint": "XZczlgiEPAQC/n86Dqa40vixNH4txnPoyuF1Q+jR6I0="
 }

packages/jsii-rosetta/lib/jsii/assemblies.ts

@@ -13,6 +13,7 @@ import {
   ApiLocation,
 } from '../snippet';
 import { enforcesStrictMode } from '../strict';
+import { mkDict } from '../util';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
 const sortJson = require('sort-json');
@@ -22,6 +23,29 @@ export interface LoadedAssembly {
   directory: string;
 }
 
+export function loadAssembliesSync(
+  assemblyLocations: readonly string[],
+  validateAssemblies: boolean,
+): readonly LoadedAssembly[] {
+  return assemblyLocations.map(loadAssemblySync);
+
+  function loadAssemblySync(location: string): LoadedAssembly {
+    const stat = fs.statSync(location);
+    if (stat.isDirectory()) {
+      return loadAssemblySync(path.join(location, '.jsii'));
+    }
+    return {
+      assembly: loadAssemblyFromFileSync(location, validateAssemblies),
+      directory: path.dirname(location),
+    };
+  }
+}
+
+function loadAssemblyFromFileSync(filename: string, validate: boolean): spec.Assembly {
+  const contents = fs.readJSONSync(filename, { encoding: 'utf-8' });
+  return validate ? spec.validateAssembly(contents) : (contents as spec.Assembly);
+}
+
 /**
  * Load assemblies by filename or directory
  */
@@ -164,3 +188,72 @@ function _fingerprint(assembly: spec.Assembly): spec.Assembly {
   const fingerprint = crypto.createHash('sha256').update(JSON.stringify(assembly)).digest('base64');
   return { ...assembly, fingerprint };
 }
+
+export interface TypeLookupAssembly {
+  readonly assembly: spec.Assembly;
+  readonly assemblyFile: string;
+  readonly symbolIdMap: Record<string, string>;
+}
+
+const MAX_ASM_CACHE = 3;
+const ASM_CACHE: TypeLookupAssembly[] = [];
+
+/**
+ * Recursively searches for a .jsii file in the directory.
+ * When file is found, checks cache to see if we already
+ * stored the assembly in memory. If not, we synchronously
+ * load the assembly into memory.
+ */
+export function findTypeLookupAssembly(directory: string): TypeLookupAssembly | undefined {
+  const pjLocation = findPackageJsonLocation(path.resolve(directory));
+  if (!pjLocation) {
+    return undefined;
+  }
+
+  const assemblyFile = path.join(path.dirname(pjLocation), '.jsii');
+
+  const fromCache = ASM_CACHE.find((c) => c.assemblyFile === assemblyFile);
+  if (fromCache) {
+    return fromCache;
+  }
+
+  if (!fs.existsSync(assemblyFile)) {
+    return undefined;
+  }
+
+  const loaded = loadLookupAssembly(assemblyFile);
+  while (ASM_CACHE.length >= MAX_ASM_CACHE) {
+    ASM_CACHE.pop();
+  }
+  ASM_CACHE.unshift(loaded);
+  return loaded;
+}
+
+function loadLookupAssembly(assemblyFile: string): TypeLookupAssembly {
+  const assembly: spec.Assembly = fs.readJSONSync(assemblyFile, { encoding: 'utf-8' });
+  const symbolIdMap = mkDict(
+    Object.values(assembly.types ?? {}).map((type) => [type.symbolId ?? '', type.fqn] as const),
+  );
+
+  return {
+    assembly,
+    assemblyFile,
+    symbolIdMap,
+  };
+}
+
+function findPackageJsonLocation(currentPath: string): string | undefined {
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    const candidate = path.join(currentPath, 'package.json');
+    if (fs.existsSync(candidate)) {
+      return candidate;
+    }
+
+    const parentPath = path.resolve(currentPath, '..');
+    if (parentPath === currentPath) {
+      return undefined;
+    }
+    currentPath = parentPath;
+  }
+}

packages/jsii-rosetta/lib/jsii/jsii-utils.ts

@@ -1,7 +1,9 @@
+import { symbolIdentifier } from 'jsii';
 import * as ts from 'typescript';
 
 import { AstRenderer } from '../renderer';
 import { typeContainsUndefined } from '../typescript/types';
+import { findTypeLookupAssembly } from './assemblies';
 import { findPackageJson } from './packages';
 
 export function isNamedLikeStruct(name: string) {
@@ -95,3 +97,88 @@ export function refersToJsiiSymbol(symbol: ts.Symbol): boolean {
   const pj = findPackageJson(declaringFile.fileName);
   return !!(pj && pj.jsii);
 }
+
+/**
+ * Returns the jsii FQN for a TypeScript (class or type) symbol
+ *
+ * TypeScript only knows the symbol NAME plus the FILE the symbol is defined
+ * in. We need to extract two things:
+ *
+ * 1. The package name (extracted from the nearest `package.json`)
+ * 2. The submodule name (...?? don't know how to get this yet)
+ * 3. Any containing type names or namespace names.
+ */
+export function jsiiFqnFromSymbol(typeChecker: ts.TypeChecker, sym: ts.Symbol): string | undefined {
+  const decl: ts.Node | undefined = sym.declarations?.[0];
+  if (!decl || !isDeclaration(decl)) {
+    return undefined;
+  }
+
+  const declSym = getSymbolFromDeclaration(decl, typeChecker);
+  if (!declSym) {
+    return undefined;
+  }
+
+  const fileName = decl.getSourceFile().fileName;
+  if (hasAnyFlag(declSym.flags, ts.SymbolFlags.Method | ts.SymbolFlags.Property | ts.SymbolFlags.EnumMember)) {
+    return fqnFromMemberSymbol(typeChecker, sym, fileName);
+  }
+  return fqnFromTypeSymbol(typeChecker, sym, fileName);
+}
+
+/**
+ * Look up the jsii fqn for a given type symbol
+ */
+function fqnFromTypeSymbol(typeChecker: ts.TypeChecker, typeSymbol: ts.Symbol, fileName: string): string | undefined {
+  const symbolId = symbolIdentifier(typeChecker, typeSymbol);
+  if (!symbolId) {
+    return undefined;
+  }
+
+  const assembly = findTypeLookupAssembly(fileName);
+  return assembly?.symbolIdMap[symbolId];
+}
+
+function fqnFromMemberSymbol(
+  typeChecker: ts.TypeChecker,
+  memberSymbol: ts.Symbol,
+  fileName: string,
+): string | undefined {
+  const declParent = memberSymbol.declarations?.[0]?.parent;
+  if (!declParent || !isDeclaration(declParent)) {
+    return undefined;
+  }
+
+  const declParentSym = getSymbolFromDeclaration(declParent, typeChecker);
+  if (!declParentSym) {
+    return undefined;
+  }
+
+  const result = fqnFromTypeSymbol(typeChecker, declParentSym, fileName);
+  return result ? `${result}#${memberSymbol.name}` : undefined;
+}
+
+function isDeclaration(x: ts.Node): x is ts.Declaration {
+  return (
+    ts.isClassDeclaration(x) ||
+    ts.isNamespaceExportDeclaration(x) ||
+    ts.isNamespaceExport(x) ||
+    ts.isModuleDeclaration(x) ||
+    ts.isEnumDeclaration(x) ||
+    ts.isEnumMember(x) ||
+    ts.isInterfaceDeclaration(x) ||
+    ts.isMethodDeclaration(x) ||
+    ts.isMethodSignature(x) ||
+    ts.isPropertyDeclaration(x) ||
+    ts.isPropertySignature(x)
+  );
+}
+
+function getSymbolFromDeclaration(decl: ts.Node, typeChecker: ts.TypeChecker): ts.Symbol | undefined {
+  if (!isDeclaration(decl)) {
+    return undefined;
+  }
+
+  const name = ts.getNameOfDeclaration(decl);
+  return name ? typeChecker.getSymbolAtLocation(name) : undefined;
+}