Fix race conditions on route 53 with retry logic

Bayer-Group · Apr 12, 2016 · eccf23e · eccf23e
1 parent 3e3c82b
commit eccf23e
Showing 1 changed file with 23 additions and 3 deletions.
diff --git a/assets/custom-types/remote-route53/remote_route53.js b/assets/custom-types/remote-route53/remote_route53.js
@@ -63,7 +63,7 @@ var createRecordSet = function(event, context) {
                 },
                 HostedZoneId: hostedZoneId
             };
-            route53.changeResourceRecordSets(params, function (err, data) {
+            changeResourceRecordSetsHelper(route53, params, function (err, data) {
                 if (err) {
                     errMsg = "create record set failed: " + err;
                     console.log(errMsg);
@@ -91,7 +91,7 @@ var updateRecordSet = function(event, context) {
                 },
                 HostedZoneId: hostedZoneId
             };
-            route53.changeResourceRecordSets(params, function (err, data) {
+            changeResourceRecordSetsHelper(route53, params, function (err, data) {
                 if (err) {
                     errMsg = "update record set failed: " + err;
                     console.log(errMsg);
@@ -120,7 +120,7 @@ var deleteRecordSet = function(event, context) {
                     },
                     HostedZoneId: hostedZoneId
                 };
-                route53.changeResourceRecordSets(params, function (err, data) {
+                changeResourceRecordSetsHelper(route53, params, function (err, data) {
                     if (err) {
                         if (err.code == "InvalidChangeBatch" && err.message && err.message.indexOf("not found") >= 0) {
                             errMsg = "WARNING: " + err;
@@ -145,6 +145,26 @@ var deleteRecordSet = function(event, context) {
     }
 };
 
+var changeResourceRecordSetsHelper = function(route53, params, callback, iteration) {
+    var iterationLimit = 100;
+    if (!iteration) {
+        iteration = 0;
+    }
+    route53.changeResourceRecordSets(params, function (err, data) {
+        if (err && err.code == "PriorRequestNotComplete") {
+            if (iteration < iterationLimit) {
+                console.log("WARNING: Prior request not complete.  Retrying");
+                changeResourceRecordSetsHelper(route53, params, callback, ++iteration, iterationLimit);
+            } else {
+                console.log("ERR: Prior request not complete, but retry limit hit.  Giving up.");
+                callback(err, data)
+            }
+        } else {
+            callback(err, data);
+        }
+    });
+};
+
 var validateCombos = function(event, context) {
     if (!event.ResourceProperties.Name) {
         errMsg = "missing parameter Name";