chore: move npm code

src/index.js

@@ -1,29 +1,17 @@
-import PouchDB from 'pouchdb-http';
 import ms from 'ms';
 import cargo from 'async/cargo.js';
 import queue from 'async/queue.js';
 import createStateManager from './createStateManager.js';
 import saveDocs from './saveDocs.js';
 import createAlgoliaIndex from './createAlgoliaIndex.js';
 import config from './config.js';
-import * as npm from './npm.js';
+import npm from './npm/index.js';
 import log from './log.js';
 import datadog from './datadog.js';
 import { loadHits } from './jsDelivr.js';
 
 log.info('🗿 npm ↔️ Algolia replication starts ⛷ 🐌 🛰');
 
-const db = new PouchDB(config.npmRegistryEndpoint, {
-  ajax: {
-    timeout: ms('2.5m'), // default is 10s
-  },
-});
-const defaultOptions = {
-  include_docs: true, // eslint-disable-line camelcase
-  conflicts: false,
-  attachments: false,
-};
-
 let loopStart = Date.now();
 
 const { index: mainIndex, client } = createAlgoliaIndex(config.indexName);
@@ -78,8 +66,7 @@ async function setSettings(index) {
 }
 
 async function logUpdateProgress(seq, nbChanges, emoji) {
-  const npmInfo = await npm.info();
-
+  const npmInfo = await npm.getInfo();
   const ratePerSecond = nbChanges / ((Date.now() - loopStart) / 1000);
   const remaining = ((npmInfo.seq - seq) / ratePerSecond) * 1000 || 0;
   log.info(
@@ -160,21 +147,17 @@ async function bootstrapLoop(lastId) {
   const start = Date.now();
   log.info('loop()', '::', lastId);
 
-  const options =
-    lastId === undefined
-      ? {}
-      : {
-          startkey: lastId,
-          skip: 1,
-        };
-
-  const start2 = Date.now();
-  const res = await db.allDocs({
-    ...defaultOptions,
-    ...options,
+  const options = {
     limit: config.bootstrapConcurrency,
+  };
+  if (lastId) {
+    options.startkey = lastId;
+    options.skip = 1;
+  }
+
+  const res = await npm.findAll({
+    options,
   });
-  datadog.timing('db.allDocs', Date.now() - start2);
 
   if (res.rows.length <= 0) {
     // Nothing left to process
@@ -220,21 +203,18 @@ async function replicate({ seq }) {
     stage: 'replicate',
   });
 
-  const { seq: npmSeqToReach } = await npm.info();
+  const { seq: npmSeqToReach } = await npm.getInfo();
   let npmSeqReached = false;
 
   return new Promise((resolve, reject) => {
-    const start2 = Date.now();
-    const changes = db.changes({
-      ...defaultOptions,
+    const listener = npm.listenToChanges({
       since: seq,
       batch_size: config.replicateConcurrency, // eslint-disable-line camelcase
       live: true,
       return_docs: false, // eslint-disable-line camelcase
     });
-    datadog.timing('db.changes', Date.now() - start2);
 
-    const q = cargo(async docs => {
+    const changesConsumer = cargo(async docs => {
       datadog.increment('packages', docs.length);
 
       try {
@@ -249,26 +229,26 @@ async function replicate({ seq }) {
       }
     }, config.replicateConcurrency);
 
-    changes.on('change', async change => {
+    listener.on('change', async change => {
       if (change.deleted === true) {
         await mainIndex.deleteObject(change.id);
         log.info(`🐌  Deleted ${change.id}`);
       }
 
-      q.push(change, err => {
+      changesConsumer.push(change, err => {
         if (err) {
           reject(err);
         }
       });
 
       if (change.seq >= npmSeqToReach) {
         npmSeqReached = true;
-        changes.cancel();
+        listener.cancel();
       }
     });
-    changes.on('error', reject);
+    listener.on('error', reject);
 
-    q.drain(() => {
+    changesConsumer.drain(() => {
       if (npmSeqReached) {
         log.info('🐌  We reached the npm current sequence');
         resolve();
@@ -287,15 +267,14 @@ async function watch({ seq }) {
   });
 
   return new Promise((resolve, reject) => {
-    const changes = db.changes({
-      ...defaultOptions,
+    const listener = npm.listenToChanges({
       since: seq,
       live: true,
       batch_size: 1, // eslint-disable-line camelcase
       return_docs: false, // eslint-disable-line camelcase
     });
 
-    const q = queue(async change => {
+    const changesConsumer = queue(async change => {
       datadog.increment('packages');
 
       try {
@@ -325,18 +304,18 @@ async function watch({ seq }) {
       }
     }, 1);
 
-    changes.on('change', async change => {
+    listener.on('change', async change => {
       if (change.deleted === true) {
         await mainIndex.deleteObject(change.id);
         log.info(`🛰 Deleted ${change.id}`);
       }
-      q.push(change, err => {
+      changesConsumer.push(change, err => {
         if (err) {
           reject(err);
         }
       });
     });
-    changes.on('error', reject);
+    listener.on('error', reject);
   });
 }
 

src/__tests__/npm.test.js → src/npm/__tests__/index.test.js

@@ -1,12 +1,12 @@
-import { info, getDownloads, getDependents } from '../npm';
+import api from '../index.js';
 
-describe('info()', () => {
+describe('getInfo()', () => {
   let registryInfo;
   beforeAll(async () => {
-    registryInfo = await info();
+    registryInfo = await api.getInfo();
   });
 
-  it('contains the correct keys', () => {
+  test('contains the correct keys', () => {
     expect(registryInfo).toEqual(
       expect.objectContaining({
         nbDocs: expect.any(Number),
@@ -19,7 +19,7 @@ describe('info()', () => {
 describe('getDependents()', () => {
   let dependents;
   beforeAll(async () => {
-    dependents = await getDependents([
+    dependents = await api.getDependents([
       { name: 'jest' },
       { name: '@angular/core' },
       { name: 'holmes.js' },
@@ -47,30 +47,42 @@ describe('getDependents()', () => {
     expect(angular).toBe(0);
     expect(holmes).toBe(0);
   });
+});
 
-  it.skip('has the right approximate value', () => {
-    const [jest, angular, holmes] = dependents.map(pkg =>
-      pkg.dependents.toString()
-    );
-
-    // eslint-disable-next-line no-console
-    console.log('dependents', { jest, angular, holmes });
-
-    // real should be 2100
-    expect(jest).toHaveLength(4);
+describe('getDownload()', () => {
+  it('should download one package and return correct response', async () => {
+    const dl = await api.getDownload('jest');
+    expect(dl.body).toHaveProperty('jest');
+    expect(dl.body.jest).toEqual({
+      downloads: expect.any(Number),
+      start: expect.any(String),
+      end: expect.any(String),
+      package: 'jest',
+    });
+  });
 
-    // real should be 5200
-    expect(angular).toHaveLength(4);
+  it('should download one scoped package and return correct response', async () => {
+    const dl = await api.getDownload('@angular/core');
+    expect(dl.body).toHaveProperty('@angular/core');
+    expect(dl.body['@angular/core']).toEqual({
+      downloads: expect.any(Number),
+      start: expect.any(String),
+      end: expect.any(String),
+      package: '@angular/core',
+    });
+  });
 
-    // real should be 0
-    expect(holmes).toHaveLength(1);
+  it('should download 2 packages and return correct response', async () => {
+    const dl = await api.getDownload('jest,holmes.js');
+    expect(dl.body).toHaveProperty('jest');
+    expect(dl.body).toHaveProperty(['holmes.js']);
   });
 });
 
 describe('getDownloads()', () => {
   let downloads;
   beforeAll(async () => {
-    downloads = await getDownloads([
+    downloads = await api.getDownloads([
       { name: 'jest' },
       { name: '@angular/core' },
       { name: 'holmes.js' },
@@ -116,9 +128,6 @@ describe('getDownloads()', () => {
       pkg.downloadsLast30Days.toString()
     );
 
-    // eslint-disable-next-line no-console
-    console.log('downloads', { jest, angular, holmes });
-
     expect(jest.length).toBeGreaterThanOrEqual(6);
     expect(jest.length).toBeLessThanOrEqual(8);