feat(typescript): pre-load definitely typed pkg (#639)

src/config.ts

@@ -152,6 +152,8 @@ export const config = {
   jsDelivrHitsEndpoint:
     'https://data.jsdelivr.com/v1/stats/packages/npm/month/all',
   jsDelivrPackageEndpoint: 'https://data.jsdelivr.com/v1/package/npm',
+  typescriptTypesIndex:
+    'https://typespublisher.blob.core.windows.net/typespublisher/data/search-index-min.json',
   unpkgRoot: 'https://unpkg.com',
   maxObjSize: 450000,
   popularDownloadsRatio: 0.005,

src/index.ts

@@ -3,6 +3,7 @@ import * as algolia from './algolia/index';
 import * as bootstrap from './bootstrap';
 import { config } from './config';
 import * as jsDelivr from './jsDelivr/index';
+import * as typescript from './typescript/index';
 import { datadog } from './utils/datadog';
 import { log } from './utils/log';
 import * as sentry from './utils/sentry';
@@ -43,6 +44,7 @@ async function main(): Promise<void> {
 
   // Preload some useful data
   await jsDelivr.loadHits();
+  await typescript.loadTypesIndex();
 
   // then we run the bootstrap
   // after a bootstrap is done, it's moved to main (with settings)

src/saveDocs.ts

@@ -7,7 +7,7 @@ import formatPkg from './formatPkg';
 import * as jsDelivr from './jsDelivr';
 import * as npm from './npm';
 import type { GetPackage } from './npm/types';
-import { getTSSupport } from './typescriptSupport';
+import { getTSSupport } from './typescript/index';
 import { datadog } from './utils/datadog';
 import { log } from './utils/log';
 

src/typescript/index.test.ts

@@ -0,0 +1,143 @@
+import * as npm from '../npm';
+import { fileExistsInUnpkg } from '../unpkg';
+
+import * as api from './index';
+
+jest.mock('../npm');
+jest.mock('../unpkg');
+
+describe('loadTypesIndex()', () => {
+  it('should download and cache all @types', async () => {
+    expect(api.typesCache).not.toHaveProperty('algoliasearch');
+    expect(api.isDefinitelyTyped({ name: 'algoliasearch' })).toBe(undefined);
+
+    await api.loadTypesIndex();
+    expect(api.typesCache).toHaveProperty('algoliasearch');
+    expect(api.typesCache).not.toHaveProperty('algoliasearch/lite');
+
+    expect(api.typesCache.algoliasearch).toBe('algoliasearch');
+    expect(api.typesCache['algoliasearch/lite']).toBe(undefined);
+    expect(api.typesCache.doesnotexist).toBe(undefined);
+
+    expect(api.isDefinitelyTyped({ name: 'algoliasearch' })).toBe(
+      'algoliasearch'
+    );
+  });
+});
+
+describe('getTypeScriptSupport()', () => {
+  it('If types are already calculated - return early', async () => {
+    const typesSupport = await api.getTypeScriptSupport({
+      name: 'Has Types',
+      types: { ts: 'included' },
+      version: '1.0',
+    });
+
+    expect(typesSupport).toEqual({ types: { ts: 'included' } });
+  });
+
+  it('Handles not having any possible TS types', async () => {
+    const typesSupport = await api.getTypeScriptSupport({
+      name: 'my-lib',
+      types: { ts: false },
+      version: '1.0',
+    });
+    expect(typesSupport).toEqual({ types: { ts: false } });
+  });
+
+  describe('Definitely Typed', () => {
+    it('Checks for @types/[name]', async () => {
+      const atTypesSupport = await api.getTypeScriptSupport({
+        name: 'lodash.valuesin',
+        types: { ts: false },
+        version: '1.0',
+      });
+      expect(atTypesSupport).toEqual({
+        types: {
+          ts: 'definitely-typed',
+          definitelyTyped: '@types/lodash.valuesin',
+        },
+      });
+    });
+
+    it('Checks for @types/[scope__name]', async () => {
+      const atTypesSupport = await api.getTypeScriptSupport({
+        name: '@mapbox/geojson-area',
+        types: { ts: false },
+        version: '1.0',
+      });
+      expect(atTypesSupport).toEqual({
+        types: {
+          ts: 'definitely-typed',
+          definitelyTyped: '@types/mapbox__geojson-area',
+        },
+      });
+
+      const atTypesSupport2 = await api.getTypeScriptSupport({
+        name: '@reach/router',
+        types: { ts: false },
+        version: '1.0',
+      });
+      expect(atTypesSupport2).toEqual({
+        types: {
+          ts: 'definitely-typed',
+          definitelyTyped: '@types/reach__router',
+        },
+      });
+    });
+  });
+
+  describe('unpkg', () => {
+    it('Checks for a d.ts resolved version of main', async () => {
+      // @ts-expect-error
+      npm.validatePackageExists.mockResolvedValue(false);
+      // @ts-expect-error
+      fileExistsInUnpkg.mockResolvedValue(true);
+
+      const typesSupport = await api.getTypeScriptSupport({
+        name: 'my-lib',
+        types: { ts: { possible: true, dtsMain: 'main.d.ts' } },
+        version: '1.0.0',
+      });
+      expect(typesSupport).toEqual({ types: { ts: 'included' } });
+    });
+  });
+
+  // TO DO : reup this
+  // adescribe('FilesList', () => {
+  //   ait('should match a correct filesList', async () => {
+  //     const atTypesSupport = await api.getTypeScriptSupport(
+  //       {
+  //         name: 'doesnotexist',
+  //         types: { ts: false },
+  //     version: '1.0',
+
+  //       },
+  //       [{ name: 'index.js' }, { name: 'index.d.ts' }]
+  //     );
+  //     expect(atTypesSupport).toEqual({
+  //       types: {
+  //         _where: 'filesList',
+  //         ts: 'included',
+  //       },
+  //     });
+  //   });
+
+  //   ait('should not match an incorrect filesList', async () => {
+  //     const atTypesSupport = await api.getTypeScriptSupport(
+  //       {
+  //         name: 'doesnotexist',
+  //         types: { ts: false },
+  //     version: '1.0',
+
+  //       },
+  //       [{ name: 'index.js' }, { name: 'index.ts' }, { name: 'index.md' }]
+  //     );
+  //     expect(atTypesSupport).toEqual({
+  //       types: {
+  //         ts: false,
+  //       },
+  //     });
+  //   });
+  // });
+});

src/typescript/index.ts

@@ -0,0 +1,108 @@
+import type { RawPkg } from '../@types/pkg';
+import { config } from '../config';
+import { fileExistsInUnpkg } from '../unpkg';
+import { datadog } from '../utils/datadog';
+import { log } from '../utils/log';
+import { request } from '../utils/request';
+
+interface TypeList {
+  p: string; // url
+  l: string; // display name
+  t: string; // package name
+  // don't known
+  d: number;
+  g: string[];
+  m: string[];
+}
+
+export const typesCache: Record<string, string> = {};
+
+/**
+ * Microsoft build a index.json with all @types/* on each publication.
+ * - https://github.com/microsoft/types-publisher/blob/master/src/create-search-index.ts.
+ */
+export async function loadTypesIndex(): Promise<void> {
+  const start = Date.now();
+  const { body } = await request<TypeList[]>(config.typescriptTypesIndex, {
+    decompress: true,
+    responseType: 'json',
+  });
+
+  log.info(`📦  Typescript preload, found ${body.length} @types`);
+
+  // m = modules associated
+  // t = @types/<name>
+  body.forEach((type) => {
+    typesCache[unmangle(type.t)] = type.t;
+  });
+
+  datadog.timing('typescript.loadTypesIndex', Date.now() - start);
+}
+
+export function isDefinitelyTyped({ name }): string | undefined {
+  return typesCache[unmangle(name)];
+}
+
+export function unmangle(name: string): string {
+  // https://github.com/algolia/npm-search/pull/407/files#r316562095
+  return name.replace('__', '/').replace('@', '');
+}
+
+/**
+ * Basically either
+ *   - { types: { ts: false }}  for no existing TypeScript support
+ *   - { types: { ts: "@types/module" }} - for definitely typed support
+ *   - { types: { ts: "included" }} - for types shipped with the module.
+ */
+export async function getTypeScriptSupport(
+  pkg: Pick<RawPkg, 'name' | 'types' | 'version'>
+): Promise<Pick<RawPkg, 'types'>> {
+  // Already calculated in `formatPkg`
+  if (pkg.types.ts === 'included') {
+    return { types: pkg.types };
+  }
+
+  // The 2nd most likely is definitely typed
+  const defTyped = isDefinitelyTyped({ name: pkg.name });
+  if (defTyped) {
+    return {
+      types: {
+        ts: 'definitely-typed',
+        definitelyTyped: `@types/${defTyped}`,
+      },
+    };
+  }
+
+  if (pkg.types.ts === false) {
+    return { types: { ts: false } };
+  }
+
+  // Do we have a main .d.ts file?
+  // TO DO: replace this with a list of files check
+  if (pkg.types.ts !== 'definitely-typed' && pkg.types.ts.possible === true) {
+    const resolved = await fileExistsInUnpkg(
+      pkg.name,
+      pkg.version,
+      pkg.types.ts.dtsMain
+    );
+    if (resolved) {
+      return { types: { ts: 'included' } };
+    }
+  }
+
+  return { types: { ts: false } };
+}
+
+/**
+ * Check if packages have Typescript definitions.
+ */
+export async function getTSSupport(
+  pkgs: Array<Pick<RawPkg, 'name' | 'types' | 'version'>>
+): Promise<Array<Pick<RawPkg, 'types'>>> {
+  const start = Date.now();
+
+  const all = await Promise.all(pkgs.map(getTypeScriptSupport));
+
+  datadog.timing('getTSSupport', Date.now() - start);
+  return all;
+}