feat: add democracylab extractor

.vscode/launch.json

@@ -24,6 +24,16 @@
                 "matchmaker",
                 "--commit-to=sites/codeforsanfrancisco.org"
             ]
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Import DemocracyLab",
+            "program": "${workspaceFolder}/import.js",
+            "args": [
+                "democracylab",
+                "--commit-to=sites/democracylab.org"
+            ]
         }
     ]
 }

import.js

@@ -3,6 +3,7 @@
 const sortKeys = require('sort-keys');
 const TOML = require('@iarna/toml');
 const axios = require('axios');
+const csvParser = require('csv-parser');
 const ProgressBar = require('progress');
 const { Repo } = require('hologit/lib');
 
@@ -13,7 +14,7 @@ require('yargs')
         builder: {
             sourceType: {
                 describe: 'Type of source being imported',
-                choices: ['laddr', 'matchmaker']
+                choices: ['laddr', 'matchmaker', 'democracylab']
             },
             source: {
                 describe: 'Host/URL for source. Format varies by source_type'
@@ -88,6 +89,8 @@ async function importTaxonomy(tree, argv) {
         return importLaddr(tree, argv);
     } else if (argv.sourceType == 'matchmaker') {
         return importMatchmaker(tree, argv);
+    } else if (argv.sourceType == 'democracylab') {
+        return importDemocracyLab(tree, argv);
     } else {
         throw new Error('Unsupported source type');
     }
@@ -175,4 +178,62 @@ async function importMatchmaker(tree, { source=null }) {
 
     // write tree
     return await tree.write();
-}
+}
+
+async function importDemocracyLab(tree, { source=null }) {
+
+    // load tags
+    if (!source) {
+        source = 'https://raw.githubusercontent.com/DemocracyLab/CivicTechExchange/master/common/models/Tag_definitions.csv';
+    }
+
+    console.warn(`downloading ${source}`);
+    const response = await axios.get(source, { responseType: 'stream' });
+
+    return new Promise ((resolve, reject) => {
+        const tags = [];
+
+        // read all tags
+        response.data
+            .pipe(csvParser())
+            .on('data', async (row) => {
+                const tagData = {};
+
+                for (const columnName in row) {
+                    const fieldName = columnName
+                        .replace(/\s*\(.*$/, '')
+                        .replace(/\s+/, '_')
+                        .toLowerCase();
+
+                    tagData[fieldName] = row[columnName];
+                }
+
+                tags.push(tagData);
+            })
+            .on('end', async () => {
+
+                // build tree
+                const progressBar = new ProgressBar('building tree :percent [:bar] :rate/s :etas', { total: tags.length });
+
+                for (const tagData of tags) {
+                    const toml = TOML.stringify(sortKeys({
+                        ...tagData,
+                        category: null,
+                        canonical_name: null,
+                        parent: tagData.parent || null,
+                        subcategory: tagData.subcategory || null,
+                        caption: tagData.caption || null
+                    }, { deep: true }));
+
+                    const blob = await tree.writeChild(`${tagData.category}/${tagData.canonical_name}.toml`, toml);
+
+                    progressBar.tick();
+                }
+
+                tree.write()
+                    .then(resolve)
+                    .catch(reject);
+            })
+            .on('error', reject)
+    });
+}

package.json

@@ -7,6 +7,7 @@
   "dependencies": {
     "@iarna/toml": "^2.2.3",
     "axios": "^0.19.0",
+    "csv-parser": "^2.3.0",
     "hologit": "^0.18.1",
     "progress": "^2.0.3",
     "sort-keys": "^3.0.0",

update-all.sh

@@ -5,3 +5,4 @@ node ./import.js laddr codeforcroatia.org --commit-to=sites/codeforcroatia.org
 node ./import.js laddr brigade.opencharlotte.org --commit-to=sites/opencharlotte.org
 node ./import.js laddr www.codeforcary.org --commit-to=sites/codeforcary.org
 node ./import.js matchmaker --commit-to=sites/codeforsanfrancisco.org
+node ./import.js democracylab --commit-to=sites/democracylab.org