diff --git a/.vscode/launch.json b/.vscode/launch.json index 82c4611..b1abfc7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -24,6 +24,16 @@ "matchmaker", "--commit-to=sites/codeforsanfrancisco.org" ] + }, + { + "type": "node", + "request": "launch", + "name": "Import DemocracyLab", + "program": "${workspaceFolder}/import.js", + "args": [ + "democracylab", + "--commit-to=sites/democracylab.org" + ] } ] } \ No newline at end of file diff --git a/import.js b/import.js index 05fdfab..847d77e 100644 --- a/import.js +++ b/import.js @@ -3,6 +3,7 @@ const sortKeys = require('sort-keys'); const TOML = require('@iarna/toml'); const axios = require('axios'); +const csvParser = require('csv-parser'); const ProgressBar = require('progress'); const { Repo } = require('hologit/lib'); @@ -13,7 +14,7 @@ require('yargs') builder: { sourceType: { describe: 'Type of source being imported', - choices: ['laddr', 'matchmaker'] + choices: ['laddr', 'matchmaker', 'democracylab'] }, source: { describe: 'Host/URL for source. Format varies by source_type' @@ -88,6 +89,8 @@ async function importTaxonomy(tree, argv) { return importLaddr(tree, argv); } else if (argv.sourceType == 'matchmaker') { return importMatchmaker(tree, argv); + } else if (argv.sourceType == 'democracylab') { + return importDemocracyLab(tree, argv); } else { throw new Error('Unsupported source type'); } @@ -175,4 +178,62 @@ async function importMatchmaker(tree, { source=null }) { // write tree return await tree.write(); -} \ No newline at end of file +} + +async function importDemocracyLab(tree, { source=null }) { + + // load tags + if (!source) { + source = 'https://raw.githubusercontent.com/DemocracyLab/CivicTechExchange/master/common/models/Tag_definitions.csv'; + } + + console.warn(`downloading ${source}`); + const response = await axios.get(source, { responseType: 'stream' }); + + return new Promise ((resolve, reject) => { + const tags = []; + + // read all tags + response.data + .pipe(csvParser()) + .on('data', async (row) => { + const tagData = {}; + + for (const columnName in row) { + const fieldName = columnName + .replace(/\s*\(.*$/, '') + .replace(/\s+/, '_') + .toLowerCase(); + + tagData[fieldName] = row[columnName]; + } + + tags.push(tagData); + }) + .on('end', async () => { + + // build tree + const progressBar = new ProgressBar('building tree :percent [:bar] :rate/s :etas', { total: tags.length }); + + for (const tagData of tags) { + const toml = TOML.stringify(sortKeys({ + ...tagData, + category: null, + canonical_name: null, + parent: tagData.parent || null, + subcategory: tagData.subcategory || null, + caption: tagData.caption || null + }, { deep: true })); + + const blob = await tree.writeChild(`${tagData.category}/${tagData.canonical_name}.toml`, toml); + + progressBar.tick(); + } + + tree.write() + .then(resolve) + .catch(reject); + }) + .on('error', reject) + }); +} diff --git a/package-lock.json b/package-lock.json index 6d81446..63e6fce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -398,6 +398,27 @@ "which": "^1.2.9" } }, + "csv-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-2.3.0.tgz", + "integrity": "sha512-yfYRZ9P9LNKuRK0lEFf40Be4HcFxe1XgxWL/QmlkakSE8SHcWbOGFZA/u7YpfGX/hVbRUdbnO5xO8XuYxrcBtA==", + "requires": { + "buffer-alloc": "^1.1.0", + "buffer-from": "^1.0.0", + "execa": "^1.0.0", + "generate-function": "^1.0.1", + "generate-object-property": "^1.0.0", + "minimist": "^1.2.0", + "ndjson": "^1.4.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + } + } + }, "cycle": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", @@ -1219,6 +1240,19 @@ } } }, + "generate-function": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-1.1.0.tgz", + "integrity": "sha1-VMIbCAGSsW2Yd3ecW7gWZudyNl8=" + }, + "generate-object-property": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "integrity": "sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA=", + "requires": { + "is-property": "^1.0.0" + } + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1777,6 +1811,11 @@ "isobject": "^3.0.1" } }, + "is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + }, "is-ssh": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/is-ssh/-/is-ssh-1.3.1.tgz", @@ -1815,6 +1854,11 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, "jsonparse": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", @@ -2006,6 +2050,24 @@ "to-regex": "^3.0.1" } }, + "ndjson": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/ndjson/-/ndjson-1.5.0.tgz", + "integrity": "sha1-rmA7NrE0vOw0e0UkIrC/mNWDLsg=", + "requires": { + "json-stringify-safe": "^5.0.1", + "minimist": "^1.2.0", + "split2": "^2.1.0", + "through2": "^2.0.3" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + } + } + }, "neo-async": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", @@ -2546,6 +2608,14 @@ "extend-shallow": "^3.0.0" } }, + "split2": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz", + "integrity": "sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==", + "requires": { + "through2": "^2.0.2" + } + }, "stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -2658,6 +2728,15 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "requires": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, "to-buffer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", diff --git a/package.json b/package.json index bea12bf..7b8514d 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "dependencies": { "@iarna/toml": "^2.2.3", "axios": "^0.19.0", + "csv-parser": "^2.3.0", "hologit": "^0.18.1", "progress": "^2.0.3", "sort-keys": "^3.0.0", diff --git a/update-all.sh b/update-all.sh index 4ec5770..cb5c194 100755 --- a/update-all.sh +++ b/update-all.sh @@ -5,3 +5,4 @@ node ./import.js laddr codeforcroatia.org --commit-to=sites/codeforcroatia.org node ./import.js laddr brigade.opencharlotte.org --commit-to=sites/opencharlotte.org node ./import.js laddr www.codeforcary.org --commit-to=sites/codeforcary.org node ./import.js matchmaker --commit-to=sites/codeforsanfrancisco.org +node ./import.js democracylab --commit-to=sites/democracylab.org