Fix some "_id" fields not being unique. (#17)

* Fix "_id" fields not being unique.

The field not being unique may cause issues with the difference checker, as it just randomly decides the order of certain objects, causing false positives in difference checkers.

- Make the "_id" fields use more than just the name, as duplicate names are possible in the cases of enemies.
- Add an "_id" field to transients and refactor the file a bit.

These are the only current files that have ambiguity, but there may be more cases of this issue possible later (think items, primarily). Might need to improve the "_id" field of those later, too. For now, however, this suffices.

* Manual regeneration of the  2018-05-17 17:00 droptables.

* Update README to add the transient "_id" field.

* Concat strings and remove redundant null JSON properties.

* Manual regeneration of the 2018-05-17 17:00 droptables.

* Resolve duplicate _id fields still occurring.

Reverts the dynamic salt in 90% of the cases, so comparisons can be more easily made. Now, when the same ID has been found in an array, try to hash it with some properties until giving up eventually and just skipping adding the object when it's the _exact_ same. This applies to enemyModTables.js and modLocations.js, as those are the currently known cases of duplicates occurring.

Also refactors the other files having inconsistent formatting.

* Manual regeneration of the 2018-05-17 17:00 droptables.

* Revert "ememy" fix for now and decrease amount of _id changes.

This reverts my previous fix for ememy > enemy for now and discards all changes to the object already in the array when all previous checks fail. This causes less removals and additions when just de-duplicating the array as the original item doesn't get edited at all.

* Manual regeneration of the 2018-05-17 17:00 droptables.

* Add modular way to check hashes of arrays.

Adds checkUniqueHash(items, source, [...properties]) function. Expects "_id" field to exist within the source object, and experts the first property to be the value that's the default hash.

* Manual regeneration of the 2018-05-17 17:00 droptables.

* var -> let

README.md

@@ -169,6 +169,7 @@ Rewards not tied to a specific location, like mods dropped by Nightmare Mode mis
 ```json
 "transientRewards": [
   {
+    "_id": "b3c734177040ffb494609a27d87d4841",
     "objectiveName": "Derelict Vault",
     "rewards": [
       {

data/info.json

@@ -1 +1 @@
-{"hash":"888d284f1b3af25d5689af8d252a3106","timestamp":1526569212093,"modified":1526569000000}
+{"hash":"888d284f1b3af25d5689af8d252a3106","timestamp":1527275015982,"modified":1526569000000}

lib/blueprintLocations.js

@@ -2,27 +2,25 @@ const {hash, parseChance} = require("./utils.js")
 
 module.exports = function($) {
     const table = $("#blueprintLocations").next("table")
-
     const tbody = table.children()['0']
 
     let item = null
-
     let items = []
 
-    for(let tr of tbody.children) {
+    for (let tr of tbody.children) {
         let elem = tr.children[0]
         let text = $(elem).text()
 
-        if(elem.name === "th" && tr.children.length === 1) {
-            if(item) {
+        if (elem.name === "th" && tr.children.length === 1) {
+            if (item) {
                 items.push(item)
             }
 
             // blueprintName is legacy, should be removed ASAP...
             item = {_id: hash(text), itemName: text, blueprintName: text, enemies: []}
         }
 
-        if(elem.name === "td" && elem.attribs.class !== "blank-row") {
+        if (elem.name === "td" && elem.attribs.class !== "blank-row") {
             let chanceElem = tr.children[2]
             let chance = parseChance($(chanceElem).text())
 
@@ -40,7 +38,7 @@ module.exports = function($) {
         }
     }
 
-    // push the last one too
+    // Push the remaining item
     items.push(item)
 
     return items

lib/cetusBountyRewards.js

@@ -2,46 +2,41 @@ const {hash, parseLocation, parseRotation, parseChance} = require("./utils.js")
 
 module.exports = function($) {
     const table = $("#cetusRewards").next("table")
-
     const tbody = table.children()['0']
 
     let bountyLevel = null
     let rotation = null
-
     let bountyRewards = []
     let bountyReward = null
-
     let stageText = null;
 
-    for(let tr of tbody.children) {
+    for (let tr of tbody.children) {
         let elem = tr.children[0]
         let text = $(elem).text()
 
-        if(elem.name === "th") {
-
+        if (elem.name === "th") {
             tmp = parseRotation(text)
 
-            if(tmp) {
+            if (tmp) {
                 rotation = tmp
             } else {
-                if(bountyReward) {
+                if (bountyReward) {
                     bountyRewards.push(bountyReward)
                 }
 
                 bountyReward = {_id: hash(text), bountyLevel: text, rewards: {A: [], B: [], C:[]}}
             }
         }
 
-        if(elem.name === "td" && elem.attribs.class !== "blank-row") {
-            if(tr.children.length === 2) {
+        if (elem.name === "td" && elem.attribs.class !== "blank-row") {
+            if (tr.children.length === 2) {
                 let stage = tr.children[1]
                 stageText = $(stage).text()
             }
 
-            if(tr.children.length === 3) {
+            if (tr.children.length === 3) {
                 let chanceElem = tr.children[2]
                 let chance = parseChance($(chanceElem).text())
-
                 text = $(tr.children[1]).text()
 
                 bountyReward.rewards[rotation].push({
@@ -55,6 +50,7 @@ module.exports = function($) {
         }
     }
 
+    // Push the remaining item
     bountyRewards.push(bountyReward)
 
     return bountyRewards

lib/enemyBlueprintTables.js

@@ -2,34 +2,30 @@ const {hash, parseChance} = require("./utils.js")
 
 module.exports = function($) {
     const table = $("#enemyBlueprintTables").next("table")
-
     const tbody = table.children()['0']
 
     let enemy = null
-
     let enemies = []
 
-    for(let tr of tbody.children) {
+    for (let tr of tbody.children) {
         let elem = tr.children[0]
         let text = $(elem).text()
 
-        if(elem.name === "th" && tr.children.length === 2) {
-            if(enemy) {
+        if (elem.name === "th" && tr.children.length === 2) {
+            if (enemy) {
                 enemies.push(enemy)
             }
 
             let itemdropchance = $(tr.children[1]).text()
-
             itemdropchance = itemdropchance.slice("Blueprint Drop Chance: ".length, itemdropchance.length - 1)
 
             // blueprintDropChance and mods are legacy, should be removed ASAP...
             enemy = {_id: hash(text), enemyName: text, enemyItemDropChance: itemdropchance, blueprintDropChance: itemdropchance, items: [], mods: []}
         }
 
-        if(elem.name === "td" && elem.attribs.class !== "blank-row") {
+        if (elem.name === "td" && elem.attribs.class !== "blank-row") {
             let chanceElem = tr.children[2]
             let chance = parseChance($(chanceElem).text())
-
             let itemName = $(tr.children[1]).text()
 
             // Legacy, should be removed ASAP...
@@ -49,7 +45,7 @@ module.exports = function($) {
         }
     }
 
-    // push the last one too
+    // Push the remaining item
     enemies.push(enemy)
 
     return enemies

lib/enemyModTables.js

@@ -1,34 +1,30 @@
-const {hash, parseChance} = require("./utils.js")
+const {hash, checkUniqueHash, parseChance} = require("./utils.js")
 
 module.exports = function($) {
     const table = $("#enemyModTables").next("table")
-
     const tbody = table.children()['0']
 
     let enemy = null
-
     let enemies = []
 
-    for(let tr of tbody.children) {
+    for (let tr of tbody.children) {
         let elem = tr.children[0]
         let text = $(elem).text()
 
-        if(elem.name === "th" && tr.children.length === 2) {
-            if(enemy) {
-                enemies.push(enemy)
+        if (elem.name === "th" && tr.children.length === 2) {
+            if (enemy) {
+                if (checkUniqueHash(enemies, enemy, ["enemyName", "ememyModDropChance", "mods"])) { enemies.push(enemy) }
             }
 
             let moddropchance = $(tr.children[1]).text()
-
             moddropchance = moddropchance.slice("Mod Drop Chance: ".length, moddropchance.length - 1)
 
             enemy = {_id: hash(text), enemyName: text, ememyModDropChance: moddropchance, enemyModDropChance: moddropchance, mods: []}
         }
 
-        if(elem.name === "td" && elem.attribs.class !== "blank-row") {
+        if (elem.name === "td" && elem.attribs.class !== "blank-row") {
             let chanceElem = tr.children[2]
             let chance = parseChance($(chanceElem).text())
-
             let modName = $(tr.children[1]).text()
 
             enemy.mods.push({
@@ -40,8 +36,8 @@ module.exports = function($) {
         }
     }
 
-    // push the last one too
-    enemies.push(enemy)
+    // Push the remaining item
+    if (checkUniqueHash(enemies, enemy, ["enemyName", "ememyModDropChance", "mods"])) { enemies.push(enemy) }
 
     return enemies
 }

lib/keyRewards.js

@@ -11,26 +11,26 @@ module.exports = function($) {
     let keys = []
     let key = null
 
-    for(let tr of tbody.children) {
+    for (let tr of tbody.children) {
         let elem = tr.children[0]
         let text = $(elem).text()
 
-        if(elem.name === "th") {
+        if (elem.name === "th") {
 
             tmp = parseRotation(text)
 
-            if(tmp) {
+            if (tmp) {
                 rotation = tmp
             } else {
-                if(key) {
+                if (key) {
                     keys.push(key)
                 }
 
                 key = {_id: hash(text), keyName: text, rewards: {A: [], B: [], C:[]}}
             }
         }
 
-        if(elem.name === "td" && elem.attribs.class !== "blank-row") {
+        if (elem.name === "td" && elem.attribs.class !== "blank-row") {
             let chanceElem = tr.children[1]
             let chance = parseChance($(chanceElem).text())
 
@@ -43,6 +43,7 @@ module.exports = function($) {
         }
     }
 
+    // Push the remaining item
     keys.push(key)
 
     return keys

lib/miscItems.js

@@ -9,12 +9,12 @@ module.exports = function($) {
 
     let enemies = []
 
-    for(let tr of tbody.children) {
+    for (let tr of tbody.children) {
         let elem = tr.children[0]
         let text = $(elem).text()
 
-        if(elem.name === "th" && tr.children.length === 2) {
-            if(enemy) {
+        if (elem.name === "th" && tr.children.length === 2) {
+            if (enemy) {
                 enemies.push(enemy)
             }
 
@@ -25,7 +25,7 @@ module.exports = function($) {
             enemy = {_id: hash(text), enemyName: text, enemyItemDropChance: itemdropchance, items: []}
         }
 
-        if(elem.name === "td" && elem.attribs.class !== "blank-row") {
+        if (elem.name === "td" && elem.attribs.class !== "blank-row") {
             let chanceElem = tr.children[2]
             let chance = parseChance($(chanceElem).text())
 
@@ -40,7 +40,7 @@ module.exports = function($) {
         }
     }
 
-    // push the last one too
+    // Push the remaining item
     enemies.push(enemy)
 
     return enemies

lib/missionRewards.js

@@ -10,21 +10,21 @@ module.exports = function($) {
 
     let missionRewards = {}
 
-    for(let tr of tbody.children) {
+    for (let tr of tbody.children) {
         let elem = tr.children[0]
         let text = $(elem).text()
 
-        if(elem.name === "th") {
+        if (elem.name === "th") {
             let tmp = parseLocation(text)
 
-            if(tmp) {
+            if (tmp) {
                 location = tmp
 
-                if(!missionRewards[location.planet]) {
+                if (!missionRewards[location.planet]) {
                     missionRewards[location.planet] = {}
                 }
 
-                if(!missionRewards[location.planet][location.location]) {
+                if (!missionRewards[location.planet][location.location]) {
                     missionRewards[location.planet][location.location] = {
                         gameMode: location.gameMode,
                         isEvent: location.isEvent,
@@ -38,17 +38,17 @@ module.exports = function($) {
             } else {
                 tmp = parseRotation(text)
 
-                if(tmp) {
+                if (tmp) {
                     rotation = tmp
                 }
             }
         }
 
-        if(elem.name === "td" && elem.attribs.class === "blank-row") {
+        if (elem.name === "td" && elem.attribs.class === "blank-row") {
             rotation = null
         }
 
-        if(elem.name === "td" && elem.attribs.class !== "blank-row") {
+        if (elem.name === "td" && elem.attribs.class !== "blank-row") {
             let chanceElem = tr.children[1]
             let chance = parseChance($(chanceElem).text())
 
@@ -59,13 +59,13 @@ module.exports = function($) {
                 chance: Number(chance.chance)
             }
 
-            if(!rotation) {
-                if(!Array.isArray(missionRewards[location.planet][location.location].rewards))
+            if (!rotation) {
+                if (!Array.isArray(missionRewards[location.planet][location.location].rewards))
                     missionRewards[location.planet][location.location].rewards = []
 
                 missionRewards[location.planet][location.location].rewards.push(item)
             } else {
-                if(!missionRewards[location.planet][location.location].rewards[rotation])
+                if (!missionRewards[location.planet][location.location].rewards[rotation])
                     missionRewards[location.planet][location.location].rewards[rotation] = []
 
                 missionRewards[location.planet][location.location].rewards[rotation].push(item)