feat(data): add jsDelivr hits (#263)

* Add jsDelivr hits

* Removed interval for re-fetching data

* Removed catch when loading jsDelivr hits

* Record schema in README.md updated

README.md

@@ -44,6 +44,7 @@ For every single NPM package, we create a record in the Algolia index. The resul
   "downloadsLast30Days": 10978749,
   "downloadsRatio": 0.08310651682685861,
   "humanDownloadsLast30Days": "11m",
+  "jsDelivrHits": 11684192,
   "popular": true,
   "version": "6.26.0",
   "versions": {

src/__tests__/__snapshots__/config.test.js.snap

@@ -82,6 +82,7 @@ Object {
       "type": "synonym",
     },
   ],
+  "jsDelivrHitsEndpoint": "https://data.jsdelivr.com/v1/stats/packages/month/all",
   "maxObjSize": 450000,
   "npmDownloadsEndpoint": "https://api.npmjs.org/downloads",
   "npmRegistryEndpoint": "https://replicate.npmjs.com/registry",

src/config.js

@@ -5,6 +5,7 @@ import ms from 'ms';
 const defaultConfig = {
   npmRegistryEndpoint: 'https://replicate.npmjs.com/registry',
   npmDownloadsEndpoint: 'https://api.npmjs.org/downloads',
+  jsDelivrHitsEndpoint: 'https://data.jsdelivr.com/v1/stats/packages/month/all',
   maxObjSize: 450000,
   popularDownloadsRatio: 0.005,
   appId: 'OFCNCOG2CU',

src/index.js

@@ -8,6 +8,7 @@ import log from './log.js';
 import ms from 'ms';
 import cargo from 'async/cargo';
 import queue from 'async/queue';
+import { loadHits } from './jsDelivr';
 
 log.info('🗿 npm ↔️ Algolia replication starts ⛷ 🐌 🛰');
 
@@ -96,6 +97,7 @@ async function bootstrap(state) {
 
   if (state.bootstrapLastId) {
     log.info('⛷ Bootstrap: starting at doc %s', state.bootstrapLastId);
+    await loadHits();
     return loop(state.bootstrapLastId);
   } else {
     const { taskID } = await client.deleteIndex(c.bootstrapIndexName);
@@ -105,6 +107,7 @@ async function bootstrap(state) {
     // first time this launches, we need to remember the last seq our bootstrap can trust
     await stateManager.save({ seq });
     await setSettings(bootstrapIndex);
+    await loadHits();
     return loop(state.bootstrapLastId);
   }
 

src/jsDelivr.js

@@ -0,0 +1,23 @@
+import got from 'got';
+import c from './config.js';
+
+const hits = new Map();
+
+function formatHits(pkg) {
+  if (pkg.type !== 'npm') {
+    return;
+  }
+
+  hits.set(pkg.name, pkg.hits);
+}
+
+export async function loadHits() {
+  const hitsJSONpromise = got(c.jsDelivrHitsEndpoint, { json: true });
+  const hitsJSON = (await hitsJSONpromise).body;
+  hits.clear();
+  hitsJSON.forEach(formatHits);
+}
+
+export function getHits(pkgs) {
+  return pkgs.map(({ name }) => ({ jsDelivrHits: hits.get(name) || 0 }));
+}

src/npm.js

@@ -14,11 +14,9 @@ export function info() {
   }));
 }
 
-const logWarning = ({ error, type, packages }) => {
+const logWarning = ({ error, type, packagesStr }) => {
   log.warn(
-    `Something went wrong asking the ${type} for \n${packages.join(
-      ','
-    )} \n${error}`
+    `Something went wrong asking the ${type} for \n${packagesStr} \n${error}`
   );
 };
 
@@ -58,7 +56,7 @@ export async function getDownloads(pkgs) {
         logWarning({
           error,
           type: 'downloads',
-          packages: pkgsNames,
+          packagesStr: pkgsNames,
         });
         return { body: {} };
       })
@@ -72,7 +70,7 @@ export async function getDownloads(pkgs) {
           logWarning({
             error,
             type: 'scoped downloads',
-            packages: [pkg],
+            packagesStr: pkg,
           });
           return { body: {} };
         })

src/saveDocs.js

@@ -2,6 +2,7 @@ import formatPkg from './formatPkg.js';
 import log from './log.js';
 import { getDownloads, getDependents } from './npm.js';
 import { getChangelogs } from './changelog.js';
+import { getHits } from './jsDelivr';
 
 export default function saveDocs({ docs, index }) {
   const rawPkgs = docs
@@ -24,17 +25,20 @@ function addMetaData(pkgs) {
     getDownloads(pkgs),
     getDependents(pkgs),
     getChangelogs(pkgs),
-  ]).then(([downloads, dependents, changelogs]) =>
+    getHits(pkgs),
+  ]).then(([downloads, dependents, changelogs, hits]) =>
     pkgs.map((pkg, index) => ({
       ...pkg,
       ...downloads[index],
       ...dependents[index],
       ...changelogs[index],
+      ...hits[index],
       _searchInternal: {
         ...pkg._searchInternal,
         ...downloads[index]._searchInternal,
         ...dependents[index]._searchInternal,
         ...changelogs[index]._searchInternal,
+        ...hits[index]._searchInternal,
       },
     }))
   );