Merge pull request #1037 from DataUSA/staging

v3.2.2 - Xenotime

api/customAttributes.js

@@ -1,5 +1,11 @@
 const axios = require("axios");
 const colors = require("../static/data/colors.json");
+const iocodes = require("../static/data/pums_naics_to_iocode.json");
+const blsInds = require("../static/data/pums_bls_industry_crosswalk.json");
+const blsOccs = require("../static/data/pums_bls_occupation_crosswalk.json");
+
+const loadJSON = require("../utils/loadJSON");
+const universitySimilar = loadJSON("/static/data/similar_universities_with_dist.json");
 
 const colorMapping = Object.keys(colors)
   .reduce((obj, k) => {
@@ -30,15 +36,15 @@ const blsIndustryMap = [
 module.exports = function(app) {
 
   const {db} = app.settings;
-  const {blsMonthlyIndustries, urls} = app.settings.cache;
+  const {blsMonthlyIndustries, urls, opeid} = app.settings.cache;
 
   app.post("/api/cms/customAttributes/:pid", async(req, res) => {
 
     const {id, dimension, hierarchy} = req.body.variables;
     const meta = await db.profile_meta.findOne({where: {dimension}}).catch(() => {});
     const {slug} = meta;
 
-    const origin = `http${ req.connection.encrypted ? "s" : "" }://${ req.headers.host }`;
+    const origin = process.env.CANON_API;
 
     const breadcrumbs = await axios.get(`${origin}/api/parents/${slug}/${id}`)
         .then(resp => resp.data)
@@ -88,10 +94,31 @@ module.exports = function(app) {
         retObj.blsMonthlyID = mapped ? mapped[1] : false;
         retObj.blsMonthlyDimension = "Supersector";
       }
+      const beaIds = iocodes[id];
+      retObj.beaL0 = beaIds && beaIds.L0 ? beaIds.L0 : false;
+      retObj.beaL1 = beaIds && beaIds.L1 ? beaIds.L1 : false;
+      retObj.blsIds = blsInds[id] || false;
+      retObj.pumsLatestYear = await axios.get(`${origin}/api/data?${hierarchy}=${id}&measures=Total%20Population&limit=1&order=Year&sort=desc`)
+          .then(resp => resp.data.data[0]["ID Year"])
+          .catch(() => 2020);
+    }
+    else if (dimension === "PUMS Occupation") {
+      retObj.blsIds = blsOccs[id] || false;
+      retObj.pumsLatestYear = await axios.get(`${origin}/api/data?${hierarchy}=${id}&measures=Total%20Population&limit=1&order=Year&sort=desc`)
+          .then(resp => resp.data.data[0]["ID Year"])
+          .catch(() => 2020);
     }
     else if (dimension === "CIP") {
       retObj.stem = id.length === 6 ? stems.includes(id) ? "Stem Major" : false : "Contains Stem Majors"
     }
+    else if (dimension === "University") {
+      const similarID = universitySimilar[id] ? universitySimilar[id].map(d => d.university) : [];
+      const similarOpeidID = similarID.map(d => opeid[d]).filter(Boolean).join(",");
+
+      retObj.opeid = opeid && opeid[id]? opeid[id] : false;
+      retObj.similarID = similarID.join(",");
+      retObj.similarOpeidId = similarOpeidID;
+    }
 
     return res.json(retObj);
 

app/d3plus.js

@@ -269,6 +269,7 @@ export default {
   },
   legendPosition: "bottom",
   legendTooltip: {
+    tbody: [],
     title(d) {
       return findTooltipTitle(d, this)
     }

app/helmet.js

@@ -4,8 +4,8 @@ const desc = "The most comprehensive visualization of U.S. public data. Data USA
 export default {
   link: [
     {rel: "icon", href: "/images/favicon.ico?v=3"},
-    {rel: "preconnect", href: "https://fonts.gstatic.com/", crossorigin: ""},
-    {rel: "stylesheet", href: "https://fonts.googleapis.com/css?family=Lato:300|Palanquin:300,400,500,600,700|Source+Sans+Pro:300,400|Pathway+Gothic+One"}
+    {rel: "preconnect", href: "https://fonts.gstatic.com/", crossorigin: "anonymous"},
+    {rel: "stylesheet", href: "https://fonts.googleapis.com/css?family=Lato:300|Palanquin:300,400,500,600,700|Source+Sans+Pro:300,400|Pathway+Gothic+One", crossorigin: "anonymous"}
   ],
   meta: [
     {charset: "utf-8"},

package.json

@@ -1,6 +1,6 @@
 {
   "name": "datausa-site",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "The most comprehensive visualization of U.S. public data",
   "main": "src/index.js",
   "scripts": {

scripts/candidateImage.js

@@ -72,7 +72,7 @@ async function printProgress() {
 
 /** */
 function getFileName(member) {
-  const {key} = member;
+  const key = member.key || member["Candidate ID"];
   return path.join(process.cwd(), `static/images/candidates/${cube}/${key}.jpg`);
 }
 

scripts/wikiCandidates.js

@@ -20,6 +20,29 @@ if (!level || !urls[level]) {
   shell.exit(1);
 }
 
+// Internal list of HTML entities for escaping.
+const entities = {
+  "&": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": "\"",
+  "&#x27;": "'",
+  "&#x60;": "`",
+  "&nbsp;": ""
+};
+
+const source = `(?:${ Object.keys(entities).join("|")  })`;
+const testRegexp = RegExp(source);
+const replaceRegexp = RegExp(source, "g");
+
+/**
+* Converts html tags to spaces, then removes redundant spaces
+*/
+function stripHTML(n) {
+  const s = String(n).replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  return testRegexp.test(s) ? s.replace(replaceRegexp, match => entities[match]) : s;
+}
+
 /** */
 async function run() {
 
@@ -36,25 +59,23 @@ async function run() {
     .querySelectorAll("th")
     .reduce((arr, d) => {
       const cols = d.rawAttrs.includes("colspan") ? +d.rawAttrs.match(/colspan="([0-9])"/)[1] : 1;
-      const text = d.querySelector("span").innerHTML
-        .replace(/\<br[\s\=\"\'A-z0-9]{1,}\/\>/g, " ");
+      const text = stripHTML(d.querySelector("span").innerHTML).replace(/\s\[[0-9].*\]/g, "");
       for (let i = 0; i < cols; i++) arr.push(text);
       return arr;
     }, []);
 
   let colOffset = 0;
   const candidates = table.querySelectorAll("tr")
     .slice(1)
+    // .slice(1, 2)
     // .slice(7, 12)
     .reduce((arr, row, ii) => {
       const obj = {};
       // console.log(row);
       row.childNodes
         .filter(d => d.nodeType !== 3)
         .forEach((column, i) => {
-          const header = headers[i + colOffset]
-            .replace(/<br[^>]*>/g, " ")
-            .replace(/\<.*$/g, "");
+          const header = headers[i + colOffset];
           let data = column;
           while (data.querySelector(".cx-segment") || data.querySelector(".cx-link")) {
             data = data.querySelector(".cx-segment") || data.querySelector(".cx-link");
@@ -76,8 +97,8 @@ async function run() {
               obj.Image = `https:${data.querySelector("img").getAttribute("src")}`;
             }
           }
-          else if (header === "Term up") {
-            obj[header] = data.innerHTML.slice(0, 4);
+          else if (header === "Class") {
+            obj["Term up"] = data.innerHTML.slice(0, 4);
           }
           else if (!["Born", "Education", "Prior experience", "Occupation(s)", "Previous office(s)", "Previous elective office(s)", "Residence"].includes(header)) {
             obj[header] = data.innerHTML

static/data/pums_naics_to_iocode.json

@@ -307,5 +307,6 @@
   "92M1": {"L1": "GFGN", "L0": "G"},
   "92M2": {"L1": "GFGN", "L0": "G"},
   "92MP": {"L1": "GFGN", "L0": "G"},
-  "9211MP": {"L1": "GSLG", "L0": "G"}
+  "9211MP": {"L1": "GSLG", "L0": "G"},
+  "11-21": {"L1": null, "L0": "11,21"}
 }