feat: refresh packages (#819)

.eslintrc

@@ -16,6 +16,18 @@
           "ts": "never"
         }
       }
-    ],
-  }
+    ]
+  },
+  "overrides": [
+    {
+      "files": [
+        "**/*.ts"
+      ],
+      "rules": {
+        "consistent-return": [
+          "off"
+        ]
+      }
+    }
+  ]
 }

src/@types/pkg.ts

@@ -43,6 +43,7 @@ export type ComputedMeta = {
 
 export interface RawPkg {
   objectID: string;
+  rev: string;
   name: string;
   downloadsLast30Days: number;
   downloadsRatio: number;

src/__tests__/__snapshots__/formatPkg.test.ts.snap

@@ -60,6 +60,7 @@ Object {
     "url": "https://github.com/algolia/npm-search",
     "user": "algolia",
   },
+  "rev": Any<String>,
   "styleTypes": Array [],
   "tags": Object {
     "latest": "1.2.3",
@@ -238,6 +239,7 @@ Are you in trouble? Read through our [contribution guidelines](https://bitbucket
     "url": "git+https://bitbucket.org/atlassian/atlaskit.git",
     "user": "atlassian",
   },
+  "rev": Any<String>,
   "styleTypes": Array [],
   "tags": Object {
     "latest": "1.6.1",
@@ -379,6 +381,7 @@ Object {
     "url": "https://github.com/atomic-package/tab",
     "user": "atomic-package",
   },
+  "rev": Any<String>,
   "styleTypes": Array [],
   "tags": Object {
     "latest": "0.0.4",
@@ -511,6 +514,7 @@ Object {
     "url": "https://github.com/algolia/create-instantsearch-app",
     "user": "algolia",
   },
+  "rev": Any<String>,
   "styleTypes": Array [],
   "tags": Object {
     "latest": "4.4.2",
@@ -599,6 +603,7 @@ index(arr, obj);
 
   MIT",
   "repository": null,
+  "rev": Any<String>,
   "styleTypes": Array [],
   "tags": Object {
     "latest": "0.0.1",
@@ -775,6 +780,7 @@ Thank you so much for contributing!!
     "url": "https://github.com/LeaVerou/prism",
     "user": "LeaVerou",
   },
+  "rev": Any<String>,
   "styleTypes": Array [
     "css",
   ],
@@ -881,6 +887,7 @@ Object {
     "url": "https://github.com/algolia/npm-search",
     "user": "algolia",
   },
+  "rev": Any<String>,
   "styleTypes": Array [],
   "tags": undefined,
   "types": Object {

src/__tests__/formatPkg.test.ts

@@ -1,7 +1,8 @@
 import NicePackage from 'nice-package';
 import isISO8601 from 'validator/lib/isISO8601.js';
 
-import formatPkg, {
+import {
+  formatPkg,
   getRepositoryInfo,
   getMains,
   getVersions,
@@ -15,7 +16,7 @@ import rawPackages from './rawPackages.json';
 const BASE: GetPackage = {
   _id: '0',
   'dist-tags': {},
-  _rev: '',
+  _rev: 'rev-1',
   name: '0',
   maintainers: [],
   readme: '',
@@ -63,6 +64,7 @@ describe('general', () => {
       .map((formattedPackage) =>
         expect(formattedPackage).toMatchSnapshot(
           {
+            rev: expect.any(String),
             lastCrawl: expect.any(String),
             _searchInternal: {
               expiresAt: expect.any(Number),
@@ -93,7 +95,7 @@ describe('general', () => {
     };
     const formatted = formatPkg(pkg);
     const postfix = ' **TRUNCATED**';
-    const ending = formatted.readme.substr(
+    const ending = formatted.readme.substring(
       formatted.readme.length - postfix.length
     );
 
@@ -103,6 +105,7 @@ describe('general', () => {
     expect(ending).toBe(postfix);
 
     expect(formatted).toMatchSnapshot({
+      rev: expect.any(String),
       readme: expect.any(String),
       lastCrawl: expect.any(String),
       _searchInternal: {
@@ -845,6 +848,7 @@ describe('deprecated', () => {
     const formatted = formatPkg(pkg);
 
     expect(formatted).toMatchSnapshot({
+      rev: expect.any(String),
       lastCrawl: expect.any(String),
       deprecated: 'Yes this is deprecated',
       isDeprecated: true,

src/__tests__/saveDocs.test.ts

@@ -1,25 +1,20 @@
 import algoliasearch from 'algoliasearch';
 
-import { saveDocs, saveDoc } from '../saveDocs';
+import { formatPkg } from '../formatPkg';
+import { saveDoc } from '../saveDocs';
 
 import preact from './preact-simplified.json';
 
 jest.setTimeout(15000);
 
-describe('batch', () => {
-  it('should be similar batch vs one', async () => {
+describe('saveDoc', () => {
+  it('should always produce the same records', async () => {
     const client = algoliasearch('e', '');
     const index = client.initIndex('a');
-    let batch;
-    let single;
-    jest.spyOn(index, 'saveObjects').mockImplementationOnce((val) => {
-      batch = val[0];
-      return true as any;
-    });
-    jest.spyOn(index, 'saveObject').mockImplementationOnce((val) => {
-      single = val;
+    jest.spyOn(index, 'saveObject').mockImplementationOnce(() => {
       return true as any;
     });
+
     const final = {
       _searchInternal: {
         alternativeNames: ['preact', 'preact.js', 'preactjs'],
@@ -221,18 +216,8 @@ describe('batch', () => {
       }),
     });
 
-    const row = { id: '', key: 'preact', value: { rev: 'a' }, doc: preact };
-    await saveDocs({ docs: [row], index });
-    await saveDoc({ row: preact, index });
+    await saveDoc({ formatted: formatPkg(preact), index });
 
-    expect(index.saveObjects).toHaveBeenCalledWith([clean]);
     expect(index.saveObject).toHaveBeenCalledWith(clean);
-    expect(single).toMatchObject({
-      ...batch,
-      lastCrawl: expect.any(String),
-      _searchInternal: {
-        expiresAt: expect.any(Number),
-      },
-    });
   });
 });

src/api.ts

@@ -5,7 +5,7 @@ import { log } from './utils/log';
 
 // Used for health check
 export function createAPI(): void {
-  const server = http.createServer((req, res) => {
+  const server = http.createServer((_req, res) => {
     datadog.check('main', datadog.CHECKS.OK);
     res.writeHead(200, { 'Content-Type': 'application/json' });
     res.end(

src/bootstrap.ts

@@ -6,6 +6,7 @@ import chalk from 'chalk';
 import type { StateManager } from './StateManager';
 import * as algolia from './algolia';
 import { config } from './config';
+import { formatPkg } from './formatPkg';
 import * as npm from './npm';
 import type { PrefetchedPkg } from './npm/Prefetcher';
 import { Prefetcher } from './npm/Prefetcher';
@@ -55,7 +56,7 @@ export class Bootstrap {
 
     if (state.seq && state.seq > 0 && state.bootstrapDone === true) {
       await algolia.putDefaultSettings(this.mainIndex, config);
-      log.info('⛷   Bootstrap: done');
+      log.info('⛷   Bootstrap: already done');
       log.info('-----');
 
       return;
@@ -118,7 +119,7 @@ export class Bootstrap {
 
     this.consumer.kill();
 
-    this.onDone();
+    await this.onDone();
   }
 
   async onDone(): Promise<void> {
@@ -192,7 +193,11 @@ function createPkgConsumer(
         return;
       }
 
-      await saveDoc({ row: res, index });
+      const formatted = formatPkg(res);
+      if (!formatted) {
+        return;
+      }
+      await saveDoc({ formatted, index });
 
       const lastId = (await stateManager.get()).bootstrapLastId;
 

src/config.ts

@@ -15,6 +15,7 @@ const indexSettings: Settings = {
     'searchable(keywords)',
     'searchable(computedKeywords)',
     'searchable(owner.name)',
+    '_searchInternal.expiresAt',
     'deprecated',
     'isDeprecated',
     'types.ts',
@@ -47,6 +48,7 @@ const indexSettings: Settings = {
   optionalWords: ['js', 'javascript'],
   separatorsToIndex: '_',
   replaceSynonymsInHighlight: false,
+  maxValuesPerFacet: 1000,
 };
 
 const indexSynonyms: Synonym[] = [
@@ -176,6 +178,7 @@ export const config = {
   retryMax: 2,
   retrySkipped: ms('1 minute'),
   retryBackoffPow: 3,
+  refreshPeriod: ms('2 minutes'),
 };
 
 export type Config = typeof config;

src/errors.ts

@@ -0,0 +1 @@
+export class DeletedError extends Error {}

src/formatPkg.ts

@@ -20,6 +20,8 @@ import type {
 } from './@types/pkg';
 import { config } from './config';
 import type { GetPackage, GetUser, PackageRepo } from './npm/types';
+import { datadog } from './utils/datadog';
+import { getExpiresAt } from './utils/getExpiresAt';
 
 const defaultGravatar = 'https://www.gravatar.com/avatar/';
 
@@ -59,11 +61,14 @@ const registrySubsetRules: Array<(pkg: NicePackageType) => Subset> = [
   }),
 ];
 
-export default function formatPkg(pkg: GetPackage): RawPkg | undefined {
+export function formatPkg(pkg: GetPackage): RawPkg | undefined {
+  const start = Date.now();
+  // Be careful NicePackage modify the Object ref
   const cleaned: NicePackageType | undefined = new NicePackage(pkg);
   if (!cleaned || !cleaned.name) {
-    return undefined;
+    return;
   }
+
   if (Array.isArray(cleaned.main)) {
     // https://github.com/angular-ui/bootstrap-bower/issues/52
     cleaned.main = cleaned.main[0];
@@ -110,7 +115,7 @@ export default function formatPkg(pkg: GetPackage): RawPkg | undefined {
   }
 
   if (!githubRepo && !lastPublisher && !author) {
-    return undefined; // ignore this package, we cannot link it to anyone
+    return; // ignore this package, we cannot link it to anyone
   }
 
   const repoInfo = getRepositoryInfo(defaultRepository);
@@ -144,6 +149,7 @@ export default function formatPkg(pkg: GetPackage): RawPkg | undefined {
 
   const rawPkg: RawPkg = {
     objectID: cleaned.name,
+    rev: cleaned.other._rev,
     name: cleaned.name,
     downloadsLast30Days: 0,
     downloadsRatio: 0,
@@ -180,13 +186,16 @@ export default function formatPkg(pkg: GetPackage): RawPkg | undefined {
     lastCrawl: new Date().toISOString(),
     _searchInternal: {
       alternativeNames,
-      expiresAt: new Date(Date.now() + config.expiresAt).getTime(),
+      expiresAt: getExpiresAt(),
     },
   };
 
   const truncated = truncatePackage(rawPkg);
 
-  return traverse(truncated).forEach(maybeEscape);
+  const escaped = traverse(truncated).forEach(maybeEscape);
+
+  datadog.timing('formatPkg', Date.now() - start);
+  return escaped;
 }
 
 function checkSize(pkg: RawPkg): {
@@ -204,7 +213,7 @@ function checkSize(pkg: RawPkg): {
   };
 }
 
-function truncatePackage(pkg: RawPkg): RawPkg | null {
+function truncatePackage(pkg: RawPkg): RawPkg | undefined {
   const smallerPkg = { ...pkg };
 
   {
@@ -225,7 +234,7 @@ function truncatePackage(pkg: RawPkg): RawPkg | null {
         '** TRUNCATED ** this package was too big, so non-essential information was removed';
       smallerPkg.versions = pkg.versions[pkg.version]
         ? {
-            [pkg.version]: pkg.versions[pkg.version],
+            [pkg.version]: pkg.versions[pkg.version]!,
           }
         : {};
       smallerPkg.tags = pkg?.tags?.latest
@@ -251,7 +260,7 @@ function truncatePackage(pkg: RawPkg): RawPkg | null {
   {
     const { isTooBig } = checkSize(smallerPkg);
     if (isTooBig) {
-      return null;
+      return;
     }
   }
 
@@ -418,8 +427,8 @@ function getGitHubRepoInfo({
   const [, user, project, path = ''] = result;
 
   return {
-    user,
-    project,
+    user: user!,
+    project: project!,
     path,
     head,
   };
@@ -461,8 +470,8 @@ function getRepositoryInfoFromHttpUrl(repository: string): Repo | null {
   return {
     url: repository,
     host: `${domain}.${domainTld}`,
-    user,
-    project,
+    user: user!,
+    project: project!,
     path,
   };
 }

src/npm/Prefetcher.ts

@@ -82,7 +82,7 @@ export class Prefetcher {
 
       this.#ready.push(...packages);
       this.#offset = offset;
-      this.#nextKey = packages[packages.length - 1].id;
+      this.#nextKey = packages[packages.length - 1]!.id;
     } catch (err) {
       sentry.report(err);
     }