fix(dev): upgrade env /2 (#595)

Co-authored-by: Haroen Viaene <hello@haroen.me>

package.json

@@ -23,21 +23,21 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@sentry/node": "5.30.0",
-    "algoliasearch": "3.35.1",
+    "@sentry/node": "6.5.1",
+    "algoliasearch": "4.9.1",
     "async": "3.2.0",
     "babel-cli": "6.26.0",
     "babel-preset-env": "1.7.0",
     "babel-preset-stage-2": "6.24.1",
     "bunyan": "1.8.15",
     "bunyan-debug-stream": "2.0.0",
-    "dotenv": "8.2.0",
+    "dotenv": "10.0.0",
     "dtrace-provider": "0.8.8",
     "escape-html": "1.0.3",
-    "got": "9.6.0",
+    "got": "11.8.2",
     "gravatar-url": "3.1.0",
     "hosted-git-info": "2.7.1",
-    "hot-shots": "6.8.7",
+    "hot-shots": "8.3.2",
     "lodash": "4.17.21",
     "ms": "2.1.3",
     "nano": "8.2.3",
@@ -49,11 +49,11 @@
     "truncate-utf8-bytes": "1.0.2"
   },
   "devDependencies": {
-    "@babel/core": "7.13.10",
-    "@babel/preset-env": "7.13.10",
+    "@babel/core": "7.14.3",
+    "@babel/preset-env": "7.14.4",
     "babel-eslint": "10.1.0",
-    "babel-jest": "24.9.0",
-    "doctoc": "1.4.0",
+    "babel-jest": "27.0.2",
+    "doctoc": "2.0.0",
     "eslint": "7.27.0",
     "eslint-config-algolia": "18.0.0",
     "eslint-config-prettier": "8.3.0",
@@ -70,7 +70,7 @@
     "pre-commit": "1.2.2",
     "prettier": "2.3.0",
     "renovate-config-algolia": "2.1.3",
-    "validator": "11.1.0"
+    "validator": "13.6.0"
   },
   "engines": {
     "node": "14.16.1",

src/algolia/index.js

@@ -51,10 +51,10 @@ async function prepare(config) {
 async function putDefaultSettings(index, config) {
   await index.setSettings(config.indexSettings);
 
-  await index.batchSynonyms(config.indexSynonyms, {
+  await index.saveSynonyms(config.indexSynonyms, {
     replaceExistingSynonyms: true,
   });
-  const { taskID } = await index.batchRules(config.indexRules, {
+  const { taskID } = await index.saveRules(config.indexRules, {
     replaceExistingRules: true,
   });
 

src/changelog.js

@@ -1,8 +1,8 @@
-import got from 'got';
 import race from 'promise-rat-race';
 
 import config from './config.js';
 import datadog from './datadog.js';
+import { request } from './utils/request.js';
 
 export const baseUrlMap = new Map([
   [
@@ -29,7 +29,7 @@ export const baseUrlMap = new Map([
 ]);
 
 async function handledGot(file) {
-  const result = await got(file, { method: 'HEAD' });
+  const result = await request(file, { method: 'HEAD' });
 
   if (
     // bitbucket returns 200 for private repos

src/config.js

@@ -19,7 +19,7 @@ const defaultConfig = {
   indexName: 'npm-search',
   bootstrapIndexName: 'npm-search-bootstrap',
   replicateConcurrency: 10,
-  bootstrapConcurrency: 100,
+  bootstrapConcurrency: 25,
   timeToRedoBootstrap: ms('2 weeks'),
   seq: null,
   indexSettings: {

src/jsDelivr/index.js

@@ -1,8 +1,7 @@
-import got from 'got';
-
 import config from '../config.js';
 import datadog from '../datadog.js';
 import log from '../log.js';
+import { request } from '../utils/request.js';
 
 const hits = new Map();
 
@@ -14,8 +13,8 @@ async function loadHits() {
   log.info('📦  Loading hits from jsDelivr');
 
   try {
-    const { body: hitsJSON } = await got(config.jsDelivrHitsEndpoint, {
-      json: true,
+    const { body: hitsJSON } = await request(config.jsDelivrHitsEndpoint, {
+      responseType: 'json',
     });
     hits.clear();
     hitsJSON.forEach((pkg) => {
@@ -81,10 +80,10 @@ async function getFilesList(pkg) {
 
   let files = [];
   try {
-    const response = await got(
+    const response = await request(
       `${config.jsDelivrPackageEndpoint}/${pkg.name}@${pkg.version}/flat`,
       {
-        json: true,
+        responseType: 'json',
       }
     );
     files = response.body.files;

src/npm/index.js

@@ -1,11 +1,11 @@
-import got from 'got';
 import chunk from 'lodash/chunk.js';
 import nano from 'nano';
 import numeral from 'numeral';
 
 import config from '../config.js';
 import datadog from '../datadog.js';
 import log from '../log.js';
+import { request } from '../utils/request.js';
 
 const registry = nano({
   url: config.npmRegistryEndpoint,
@@ -97,8 +97,8 @@ async function getInfo() {
 
   const {
     body: { doc_count: nbDocs, update_seq: seq },
-  } = await got(config.npmRegistryEndpoint, {
-    json: true,
+  } = await request(config.npmRegistryEndpoint, {
+    responseType: 'json',
   });
 
   datadog.timing('npm.info', Date.now() - start);
@@ -119,8 +119,7 @@ async function validatePackageExists(pkgName) {
 
   let exists;
   try {
-    const response = await got(`${config.npmRootEndpoint}/${pkgName}`, {
-      json: true,
+    const response = await request(`${config.npmRootEndpoint}/${pkgName}`, {
       method: 'HEAD',
     });
     exists = response.statusCode === 200;
@@ -153,8 +152,8 @@ function getDependents(pkgs) {
 async function getTotalDownloads() {
   const {
     body: { downloads: totalNpmDownloadsPerDay },
-  } = await got(`${config.npmDownloadsEndpoint}/range/last-month`, {
-    json: true,
+  } = await request(`${config.npmDownloadsEndpoint}/range/last-month`, {
+    responseType: 'json',
   });
 
   return totalNpmDownloadsPerDay.reduce(
@@ -170,10 +169,10 @@ async function getTotalDownloads() {
  */
 async function getDownload(pkgNames) {
   try {
-    const response = await got(
+    const response = await request(
       `${config.npmDownloadsEndpoint}/point/last-month/${pkgNames}`,
       {
-        json: true,
+        responseType: 'json',
       }
     );
     if (response.body.downloads) {

src/saveDocs.js

@@ -35,7 +35,7 @@ export default async function saveDocs({ docs, index }) {
   datadog.timing('saveDocs.addMetaData', Date.now() - start2);
 
   start2 = Date.now();
-  index.saveObjects(pkgs);
+  await index.saveObjects(pkgs);
   datadog.timing('saveDocs.saveObjects', Date.now() - start2);
 
   datadog.timing('saveDocs', Date.now() - start);

src/unpkg.js

@@ -1,14 +1,13 @@
-import got from 'got';
-
 import config from './config.js';
+import { request } from './utils/request.js';
 
 // make a head request to a route like:
 // https://unpkg.com/lodash@4.17.11/_LazyWrapper.js
 // to validate the existence of a particular file
 export async function fileExistsInUnpkg(pkg, version, path) {
   const uri = `${config.unpkgRoot}/${pkg}@${version}/${path}`;
   try {
-    const response = await got(uri, {
+    const response = await request(uri, {
       method: 'HEAD',
     });
     return response.statusCode === 200;

src/utils/request.js

@@ -0,0 +1,25 @@
+import http from 'http';
+import https from 'https';
+
+import got from 'got';
+
+const options = {
+  keepAlive: true,
+  timeout: 60000,
+  maxFreeSockets: 2000,
+  scheduling: 'fifo',
+};
+
+// The agents will pool TCP connections
+export const httpAgent = new http.Agent(options);
+export const httpsAgent = new https.Agent(options);
+
+export async function request(url, opts) {
+  return await got(url, {
+    ...opts,
+    agent: {
+      http: httpAgent,
+      https: httpsAgent,
+    },
+  });
+}