Add support for distributed caching

mediocre · Jul 5, 2019 · 0cd4fff · 0cd4fff
1 parent 9f2ad3d
commit 0cd4fff
Show file tree

Hide file tree

Showing 8 changed files with 94 additions and 38 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -15,5 +15,8 @@ notifications:
       - secure: "SOyb1L7KTf/NxLj+lunFnq4UJxXVIHHwFdPwIe4c3VbBZOeUR/0x+b2swNgJ+/rnra0Th+RCZTZ99LrBGpFchFI6hYXjLVdGGL8LhyfUuREG2tW7/MsQfTQuZwqqc6cc0trn4VGSJYVHDW428XUPc3VIJyrV1KP8NBsmMl499AY="
     on_success: always
 
+services:
+  - redis-server
+
 script:
   - npm run coveralls
diff --git a/carriers/pitneyBowes.js b/carriers/pitneyBowes.js
@@ -46,7 +46,7 @@ function PitneyBowes(options) {
 
             // Lookup each location
             async.mapLimit(locations, 10, function(location, callback) {
-                geography.parseLocation(location, function(err, address) {
+                geography.parseLocation(location, options, function(err, address) {
                     if (err || !address) {
                         return callback(err, address);
                     }

diff --git a/carriers/usps.js b/carriers/usps.js
@@ -122,8 +122,8 @@ function USPS(options) {
                 const locations = Array.from(new Set(scanDetailsList.map(scanDetail => scanDetail.location)));
 
                 // Lookup each location
-                async.mapLimit(locations, 10, function (location, callback) {
-                    geography.parseLocation(location, function (err, address) {
+                async.mapLimit(locations, 10, function(location, callback) {
+                    geography.parseLocation(location, options, function(err, address) {
                         if (err || !address) {
                             return callback(err, address);
                         }
@@ -132,7 +132,7 @@ function USPS(options) {
 
                         callback(null, address);
                     });
-                }, function (err, addresses) {
+                }, function(err, addresses) {
                     if (err) {
                         return callback(err);
                     }

diff --git a/index.js b/index.js
@@ -3,6 +3,16 @@ const FedEx = require('./carriers/fedEx');
 const USPS = require('./carriers/usps');
 
 function Bloodhound(options) {
+    // Allow PitneyBowes to cache geocode results in Redis (via petty-cache)
+    if (options && options.pettyCache && options.pitneyBowes) {
+        options.pitneyBowes.pettyCache = options.pettyCache;
+    }
+
+    // Allow USPS to cache geocode results in Redis (via petty-cache)
+    if (options && options.pettyCache && options.usps) {
+        options.usps.pettyCache = options.pettyCache;
+    }
+
     const fedEx = new FedEx(options && options.fedEx);
     const pitneyBowes = new PitneyBowes(options && options.pitneyBowes);
     const usps = new USPS(options && options.usps);

diff --git a/package.json b/package.json
@@ -4,6 +4,7 @@
         "async": "~3.1.0",
         "moment-timezone": "~0.5.25",
         "node-geocoder": "~3.23.0",
+        "petty-cache": "~2.4.1",
         "pitney-bowes": "~0.1.0",
         "shipping-fedex": "0.2.0",
         "tz-lookup": "~6.1.18",
@@ -35,5 +36,5 @@
         "type": "git",
         "url": "https://github.com/mediocre/bloodhound.git"
     },
-    "version": "0.2.2"
+    "version": "0.3.0"
 }
diff --git a/test/carriers/pitneyBowes.js b/test/carriers/pitneyBowes.js
@@ -7,6 +7,7 @@ describe('Newgistics', function() {
 
     it('should return an error for invalid base URL', function(done) {
         const bloodhound = new Bloodhound({
+            pettyCache: {},
             pitneyBowes: {
                 baseUrl: 'https://httpbin.org/status/500#'
             }

diff --git a/test/util/geography.js b/test/util/geography.js
@@ -84,4 +84,21 @@ describe('geography.parseLocation', function() {
             done();
         });
     });
+});
+
+describe('petty-cache', function() {
+    it('should store results in Redis (via petty-cache)', function(done) {
+        geography.parseLocation('Carrollton, TX', { pettyCache: {} }, function(err, actual) {
+            assert.ifError(err);
+
+            const expected = {
+                city: 'Carrollton',
+                state: 'TX',
+                timezone: 'America/Chicago'
+            };
+
+            assert.deepStrictEqual(actual, expected);
+            done();
+        });
+    });
 });
diff --git a/util/geography.js b/util/geography.js
@@ -1,39 +1,59 @@
 const async = require('async');
 const NodeGeocoder = require('node-geocoder');
 const normalize = require('us-states-normalize');
+const PettyCache = require('petty-cache');
 const tzlookup = require('tz-lookup');
 
 const geocoder = NodeGeocoder({ provider: 'openstreetmap' });
+var pettyCache;
 
 function geocode(location, callback) {
-    // Geocode the location
-    async.retry(function(callback) {
-        geocoder.geocode(location, callback);
+    async.auto({
+        geocode: function(callback) {
+            // Geocode the location
+            async.retry(function(callback) {
+                geocoder.geocode(location, callback);
+            }, function(err, results) {
+                if (err) {
+                    return callback(err);
+                }
+
+                if (!results.length) {
+                    return callback();
+                }
+
+                const firstResult = results[0];
+
+                // Check to see if the first result has the data we need
+                if (firstResult.city && firstResult.state && firstResult.zipcode) {
+                    return callback(null, firstResult);
+                }
+
+                // Reverse geocode to ensure we get a city, state, and zip
+                async.retry(function(callback) {
+                    geocoder.reverse({ lat: firstResult.latitude, lon: firstResult.longitude }, callback);
+                }, function(err, results) {
+                    if (err) {
+                        return callback(err);
+                    }
+
+                    callback(null, results[0]);
+                });
+            });
+        }
     }, function(err, results) {
         if (err) {
             return callback(err);
         }
 
-        if (!results.length) {
+        if (!results.geocode) {
             return callback();
         }
 
-        const firstResult = results[0];
-
-        // Check to see if the first result has the data we need
-        if (firstResult.city && firstResult.state && firstResult.zipcode) {
-            return callback(null, firstResult);
-        }
-
-        // Reverse geocode to ensure we get a city, state, and zip
-        async.retry(function(callback) {
-            geocoder.reverse({ lat: firstResult.latitude, lon: firstResult.longitude }, callback);
-        }, function(err, results) {
-            if (err) {
-                return callback(err);
-            }
-
-            callback(null, results[0]);
+        callback(null, {
+            city: results.geocode.city,
+            state: normalize(results.geocode.state),
+            timezone: tzlookup(results.geocode.latitude, results.geocode.longitude)
         });
     });
 }
@@ -68,20 +88,24 @@ exports.addressToString = function(address) {
     return value.trim();
 };
 
-exports.parseLocation = async.memoize(function(location, callback) {
-    geocode(location, function(err, result) {
-        if (err) {
-            return callback(err);
-        }
+exports.parseLocation = async.memoize(function(location, options, callback) {
+    // Options are optional
+    if (typeof options === 'function') {
+        callback = options;
+        options = undefined;
+    }
 
-        if (!result) {
-            return callback();
+    // Use Redis (via Petty Cache)
+    if (options && options.pettyCache) {
+        if (!pettyCache) {
+            pettyCache = new PettyCache(options.pettyCache.port, options.pettyCache.host, options.pettyCache.options);
         }
 
-        callback(null, {
-            city: result.city,
-            state: normalize(result.state),
-            timezone: tzlookup(result.latitude, result.longitude)
-        });
-    });
+        // Cache between 1-2 months
+        return pettyCache.fetch(`bloodhound.geocode:${location}`, function(callback) {
+            geocode(location, callback);
+        }, { ttl: { min: 2592000000, max: 5184000000 } }, callback);
+    }
+
+    geocode(location, callback);
 });