feat(ranking): do tie breaking based on the magnitude of downloads (#178

)

Doing tie-breaking on the magnitue of the downloads (length of the number) allows for us to put other rules that simply "oh this one is 1 time more downloaded so it should be first"

This also introduced the key `_searchInternal` for data which isn't supposed to be rendered ever. I could put them in `unretrievableAttributes`, but that might be overkill.

We can test this by doing seq=0 and seeing what is happening in the bootstrap index, except if someone else has a better idea of testing it

README.md

@@ -117,11 +117,14 @@ For every single NPM package, we create a record in the Algolia index. The resul
     // [...]
   ],
   "lastCrawl": "2017-10-24T08:29:24.672Z",
-  "popularName": "babel-core",
   "dependents": 3321,
   "humanDependents": "3.3k",
   "changelogFilename": null, // if babel-core had a changelog, it would be the raw GitHub url here
-  "objectID": "babel-core"
+  "objectID": "babel-core",
+  "_searchInternal": {
+    "popularName": "babel-core",
+    "downloadsMagnitude": 8
+  }
 }
 ```
 
@@ -135,7 +138,7 @@ If you want to learn more about how Algolia's ranking algorithm is working, you
 
 We're restricting the search to use a subset of the attributes only:
 
-* `popularName`
+* `_searchInternal.popularName`
 * `name`
 * `description`
 * `keywords`
@@ -237,9 +240,10 @@ When the process starts with `seq=0`:
 * watch for registry changes continuously and replicate them
 
 Replicate and watch are separated because:
-1. In replicate we want to replicate a batch of documents in a fast way
-2. In watch we want new changes as fast as possible, one by one. If watch was
-asking for batches of 100, new packages would be added too late to the index
+
+1.  In replicate we want to replicate a batch of documents in a fast way
+2.  In watch we want new changes as fast as possible, one by one. If watch was
+    asking for batches of 100, new packages would be added too late to the index
 
 ## Tests
 

src/__tests__/__snapshots__/config.test.js.snap

@@ -11,12 +11,12 @@ Object {
     Object {
       "condition": Object {
         "anchoring": "is",
-        "pattern": "{facet:concatenatedName}",
+        "pattern": "{facet:_searchInternal.concatenatedName}",
       },
       "consequence": Object {
         "params": Object {
           "automaticOptionalFacetFilters": Array [
-            "concatenatedName",
+            "_searchInternal.concatenatedName",
           ],
         },
       },
@@ -26,13 +26,15 @@ Object {
   ],
   "indexSettings": Object {
     "attributesForFaceting": Array [
-      "filterOnly(concatenatedName)",
+      "filterOnly(_searchInternal.concatenatedName)",
       "searchable(keywords)",
       "searchable(computedKeywords)",
       "searchable(owner.name)",
       "deprecated",
     ],
     "customRanking": Array [
+      "desc(_searchInternal.downloadsMagnitude)",
+      "desc(dependents)",
       "desc(downloadsLast30Days)",
     ],
     "disableExactOnAttributes": Array [
@@ -68,9 +70,9 @@ Object {
     ],
     "replaceSynonymsInHighlight": false,
     "searchableAttributes": Array [
-      "unordered(popularName)",
+      "unordered(_searchInternal.popularName)",
       "unordered(name)",
-      "unordered(concatenatedName)",
+      "unordered(_searchInternal.concatenatedName)",
       "unordered(description)",
       "unordered(keywords)",
       "owner.name",

src/__tests__/formatPkg.test.js

@@ -24,7 +24,7 @@ it('truncates long readmes', () => {
     formatted.readme.length - truncatedEnding.length
   );
 
-  expect(formatted.readme).toHaveLength(451134);
+  expect(formatted.readme).toHaveLength(451140);
   expect(ending).toBe(truncatedEnding);
 
   formatted.lastCrawl = '<!-- date replaced -->';

src/__tests__/npm.test.js

@@ -76,27 +76,36 @@ describe('getDownloads()', () => {
       expect.objectContaining({
         downloadsLast30Days: expect.any(Number),
         downloadsRatio: expect.any(Number),
-        popular: true,
-        popularName: 'jest',
         humanDownloadsLast30Days: expect.any(String),
+        popular: true,
+        _searchInternal: {
+          popularName: 'jest',
+          downloadsMagnitude: expect.any(Number),
+        },
       }),
       expect.objectContaining({
         downloadsLast30Days: expect.any(Number),
         downloadsRatio: expect.any(Number),
-        popular: true,
-        popularName: '@angular/core',
         humanDownloadsLast30Days: expect.any(String),
+        popular: true,
+        _searchInternal: {
+          popularName: '@angular/core',
+          downloadsMagnitude: expect.any(Number),
+        },
       }),
       expect.objectContaining({
         downloadsLast30Days: expect.any(Number),
         downloadsRatio: expect.any(Number),
-        popular: false,
         humanDownloadsLast30Days: expect.any(String),
+        popular: false,
+        _searchInternal: {
+          downloadsMagnitude: expect.any(Number),
+        },
       }),
     ]);
   });
 
-  it('has the right approximate value', () => {
+  it('has the right approximate value for downloadsLast30Days', () => {
     const [jest, angular, holmes] = downloads.map(
       pkg => pkg.downloadsLast30Days
     );
@@ -113,4 +122,14 @@ describe('getDownloads()', () => {
     expect(holmes).toBeGreaterThan(250);
     expect(holmes).toBeLessThan(550);
   });
+
+  it('has the right approximate value for downloadsMagnitude', () => {
+    const [jest, angular, holmes] = downloads.map(
+      pkg => pkg._searchInternal.downloadsMagnitude
+    );
+
+    expect(jest).toBe(7);
+    expect(angular).toBe(7);
+    expect(holmes).toBe(3);
+  });
 });

src/config.js

@@ -16,22 +16,26 @@ const defaultConfig = {
   seq: null,
   indexSettings: {
     searchableAttributes: [
-      'unordered(popularName)',
+      'unordered(_searchInternal.popularName)',
       'unordered(name)',
-      'unordered(concatenatedName)',
+      'unordered(_searchInternal.concatenatedName)',
       'unordered(description)',
       'unordered(keywords)',
       'owner.name',
       'owners.name',
     ],
     attributesForFaceting: [
-      'filterOnly(concatenatedName)' /* optionalFacetFilters to boost the name */,
+      'filterOnly(_searchInternal.concatenatedName)' /* optionalFacetFilters to boost the name */,
       'searchable(keywords)',
       'searchable(computedKeywords)',
       'searchable(owner.name)',
       'deprecated',
     ],
-    customRanking: ['desc(downloadsLast30Days)'],
+    customRanking: [
+      'desc(_searchInternal.downloadsMagnitude)',
+      'desc(dependents)',
+      'desc(downloadsLast30Days)',
+    ],
     disablePrefixOnAttributes: ['keywords', 'owner.name', 'owners.name'],
     disableExactOnAttributes: [
       'description',
@@ -69,12 +73,12 @@ const defaultConfig = {
       objectID: 'promote-exact',
       description: 'promote exact matches',
       condition: {
-        pattern: '{facet:concatenatedName}',
+        pattern: '{facet:_searchInternal.concatenatedName}',
         anchoring: 'is',
       },
       consequence: {
         params: {
-          automaticOptionalFacetFilters: ['concatenatedName'],
+          automaticOptionalFacetFilters: ['_searchInternal.concatenatedName'],
         },
       },
     },

src/formatPkg.js

@@ -52,7 +52,6 @@ export default function formatPkg(pkg) {
       : null;
 
   const owner = getOwner(repository, lastPublisher, author); // always favor the repository owner
-  const badPackage = isBadPackage(owner);
   const { computedKeywords, computedMetadata } = getComputedData(cleaned);
   const keywords = getKeywords(cleaned);
 
@@ -65,7 +64,6 @@ export default function formatPkg(pkg) {
   const rawPkg = {
     objectID: cleaned.name,
     name: cleaned.name,
-    concatenatedName,
     downloadsLast30Days: 0,
     downloadsRatio: 0,
     humanDownloadsLast30Days: numeral(0).format('0.[0]a'),
@@ -83,7 +81,6 @@ export default function formatPkg(pkg) {
     readme: pkg.readme,
     owner,
     deprecated: cleaned.deprecated !== undefined ? cleaned.deprecated : false,
-    badPackage,
     homepage: getHomePage(cleaned.homepage, cleaned.repository),
     license,
     keywords,
@@ -94,6 +91,9 @@ export default function formatPkg(pkg) {
     lastPublisher,
     owners: (cleaned.owners || []).map(formatUser),
     lastCrawl: new Date().toISOString(),
+    _searchInternal: {
+      concatenatedName,
+    },
   };
 
   const totalSize = sizeof(rawPkg);
@@ -108,16 +108,6 @@ export default function formatPkg(pkg) {
   return traverse(rawPkg).forEach(maybeEscape);
 }
 
-function isBadPackage(owner) {
-  // the organisations `npmtest` and `npmdoc` are just republishing everything
-  // this is a total mess, and shouldn't ever be used, because it makes results
-  // messy. We're ignoring packages by those "authors"
-  if (owner === 'npmdoc' || owner === 'npmtest') {
-    return true;
-  }
-  return false;
-}
-
 function maybeEscape(node) {
   if (this.isLeaf && typeof node === 'string') {
     if (this.key === 'readme') {

src/npm.js

@@ -96,16 +96,20 @@ export async function getDownloads(pkgs) {
       : 0;
     const downloadsRatio = downloadsLast30Days / totalNpmDownloads * 100;
     const popular = downloadsRatio > c.popularDownloadsRatio;
-    // if the package is popular, we copy its name to a dedicated attribute
-    // which will make popular records' `name` matches to be ranked higher than other matches
-    // see the `searchableAttributes` index setting
-    const popularAttributes = popular ? { popularName: name } : {};
+    const downloadsMagnitude = downloadsLast30Days.toString().length;
+
     return {
-      ...popularAttributes,
       downloadsLast30Days,
       humanDownloadsLast30Days: numeral(downloadsLast30Days).format('0.[0]a'),
       downloadsRatio,
       popular,
+      _searchInternal: {
+        // if the package is popular, we copy its name to a dedicated attribute
+        // which will make popular records' `name` matches to be ranked higher than other matches
+        // see the `searchableAttributes` index setting
+        ...(popular && { popularName: name }),
+        downloadsMagnitude,
+      },
     };
   });
 }

src/saveDocs.js

@@ -30,6 +30,12 @@ function addMetaData(pkgs) {
       ...downloads[index],
       ...dependents[index],
       ...changelogs[index],
+      _searchInternal: {
+        ...pkg._searchInternal,
+        ...downloads[index]._searchInternal,
+        ...dependents[index]._searchInternal,
+        ...changelogs[index]._searchInternal,
+      },
     }))
   );
 }