diff --git a/bin/csd-scraper.js b/bin/csd-scraper.js new file mode 100644 index 0000000..2ea5033 --- /dev/null +++ b/bin/csd-scraper.js @@ -0,0 +1,63 @@ +/* eslint-disable array-callback-return */ +/* eslint-disable prefer-destructuring */ +/* eslint-disable no-plusplus */ +/* eslint-disable prefer-arrow-callback */ +/* eslint-disable func-names */ +/* eslint-disable prefer-template */ +const fs = require('fs') +const casper = require('casper').create({ + pageSettings: { + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:23.0) Gecko/20130404 Firefox/23.0' + }, + verbose: false, + logLevel: 'debug' +}) + +// https://www.methodsandtools.com/tools/casperjs.php +const baseUrl = 'https://services.cds.ca/applications/taxforms/taxforms.nsf' +const url = baseUrl + '/PROCESSED-EN-' +const params = 'OpenView&Start=1&Count=3000&RestrictToCategory=All-2018' + +casper.start(url + '?' + params, function () { + this.echo('Clicking I accept link') + this.clickLabel('I accept', 'a') +}) + +casper.then(function () { + const tableId = '#taxlist' + this.echo('Loading table data ...') + if (this.exists(tableId)) { + this.echo('Found ' + tableId + ' table', 'INFO') + } else { + this.echo(tableId + ' not found', 'ERROR') + } + + const data = this.evaluate(function () { + const tableRows = document.querySelectorAll('#taxlist > tbody > tr') + return Array.prototype.map.call(tableRows, function (row) { + return { + date: row.querySelector('td:nth-child(1) span.Date').textContent, + // company: row.querySelector('td:nth-child(4) a').textContent, + docType: row.querySelector('td:nth-child(5) span.Type').textContent, + docUrl: row.querySelector('td:nth-child(6) a').href + } + }) + }) + fs.write('downloads/data.js', JSON.stringify(data), 'w') + + data.map(function (d, index) { + console.log(index + 1 + '/' + data.length) + const docUrl = data[index].docUrl + const filename = docUrl.substr(docUrl.lastIndexOf('/') + 1) + casper.download(docUrl, 'downloads/' + filename) + }) +}) + +casper.options.onResourceRequested = function (_casper, requestData) { + console.log('[LOAD]', requestData.url) +} + +casper.run(function () { + this.echo('Done') + this.exit() +}) diff --git a/bin/xsl-scraper.js b/bin/xsl-scraper.js new file mode 100644 index 0000000..1ea147f --- /dev/null +++ b/bin/xsl-scraper.js @@ -0,0 +1,146 @@ +/* eslint-disable no-plusplus */ +/* eslint-disable camelcase */ +const xlsx = require('xlsx') +const fs = require('fs') +const path = require('path') + +const DOCUMENT_ROOT = path.resolve(process.cwd(), 'downloads') + +// Set this to null to parse all documents +// Set this to a filename to parse only that document +const PARSE_SINGLE_DOCUMENT = null +// const PARSE_SINGLE_DOCUMENT = 'CDSP-AWES3N_T5013_R15_TY2018_2018_02_28_15_31_24.xls' +const PARSE_ONLY_DOCUMENT_TYPE = 'T3' + +// eslint-disable-next-line camelcase +const parse_T3 = (ws) => { + console.log('parse_T3') + // console.log(ws) + const distributionColumns = Array.from({ length: 14 }, (_, i) => String.fromCharCode('D'.charCodeAt(0) + i)) + let count = 0 + let column = distributionColumns[count] + const distributions = [] + while (ws[`${column}19`]) { + distributions.push({ + totalAmountPerUnit: ws[`${column}19`].v, + paymentDate: ws[`${column}21`].v, + capitalGains: ws[`${column}25`] ? ws[`${column}25`].v : undefined, + amountOfEligibleDividends: ws[`${column}26`] ? ws[`${column}26`].v : undefined, + amountOfNonEligibleDividends: ws[`${column}27`] ? ws[`${column}27`].v : undefined, + foreignBusinessIncome: ws[`${column}28`] ? ws[`${column}28`].v : undefined, + foreignNonBusinessIncome: ws[`${column}29`] ? ws[`${column}29`].v : undefined, + otherInvestmentIncome: ws[`${column}30`] ? ws[`${column}30`].v : undefined, + returnOfCapital: ws[`${column}32`] ? ws[`${column}32`].v : undefined + }) + count++ + column = distributionColumns[count] + } + + return { + year: ws.A2.v, + name: ws.C5.v, + symbol: ws.M5.v, + website: ws.M6 ? ws.M6.v : undefined, + calculationMethod: ws.G15.v === 1 ? 'RATE' : 'PER CENT', + distributions + } +} + +// eslint-disable-next-line camelcase +const parse_T5013_R5 = (ws) => { + console.log('parse_T5013_R5') + const distributionColumns = Array.from({ length: 14 }, (_, i) => String.fromCharCode('E'.charCodeAt(0) + i)) + let count = 0 + let column = distributionColumns[count] + const distributions = [] + while (ws[`${column}21`]) { + distributions.push({ + paymentDate: ws[`${column}21`].v, + + returnOfCapital: ws[`${column}42`] ? ws[`${column}42`].v : undefined + + }) + count++ + column = distributionColumns[count] + } + return { + year: ws.A2.v, + name: ws.C7.v, + website: ws.M13 ? ws.M13.v : undefined, + calculationMethod: ws.P5.v === 1 ? 'YEAR-END POSITION' : 'DISTRIBUTION', + distributions + } +} + +// eslint-disable-next-line camelcase +const parse_T5 = (ws) => { + console.log('parse_T5') + + // console.log(ws['!objects']) + return { + year: 1 + } +} + +// eslint-disable-next-line camelcase +const parse_T3_French = (ws) => { + console.log('parse_T3_French') + return { + year: ws.A2.v, + name: ws.C5.v, + symbol: ws.M5.v, + website: ws.M6 ? ws.M6.v : undefined, + calculationMethod: ws.G15.v === 1 ? 'RATE' : 'PER CENT' + } +} + +const parseXlsDocument = (folder, filename) => { + const wb = xlsx.readFile(`${folder}/${filename}`) + const ws = wb.Sheets[wb.SheetNames[0]] + let parsingFunction + switch (ws.A1.v) { + case 'T5013/R15 Form': + if (PARSE_ONLY_DOCUMENT_TYPE && PARSE_ONLY_DOCUMENT_TYPE === 'T5013') { + parsingFunction = parse_T5013_R5 + } + break + case 'Statement of Split Shares Income Allocations and Designations': + if (PARSE_ONLY_DOCUMENT_TYPE && PARSE_ONLY_DOCUMENT_TYPE === 'T5') { + parsingFunction = parse_T5 + } + break + case 'Statement of Trust Income Allocations and Designations': + case 'ETAT DES REVENUS DE FIDUCIE (REPARTITION ET ALLOCATION)': + if (PARSE_ONLY_DOCUMENT_TYPE && PARSE_ONLY_DOCUMENT_TYPE === 'T3') { + parsingFunction = parse_T3 + } + break + default: + console.log('Unknown xsl document type') + console.log(ws.A1) + process.exit() + } + + if (parsingFunction) { + console.log('Parsing: ', filename) + const data = parsingFunction(ws) + console.log(data) + if (!data.name) { + console.log('ERROR: unexpected data') + process.exit() + } + } +} + +if (PARSE_SINGLE_DOCUMENT) { + parseXlsDocument(DOCUMENT_ROOT, PARSE_SINGLE_DOCUMENT) +} else { + fs.readdir(DOCUMENT_ROOT, (err, list) => { + if (err) throw err + list.forEach((filename) => { + if (path.extname(filename) === '.xls') { + parseXlsDocument(DOCUMENT_ROOT, filename) + } + }) + }) +} diff --git a/package-lock.json b/package-lock.json index b5e3e86..c4ec52e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2127,6 +2127,15 @@ } } }, + "adler-32": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.2.0.tgz", + "integrity": "sha1-aj5r8KY5ALoVZSgIyxXGgT0aXyU=", + "requires": { + "exit-on-epipe": "~1.0.1", + "printj": "~1.1.0" + } + }, "aggregate-error": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.0.1.tgz", @@ -2420,19 +2429,9 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, "attr-accept": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-1.1.3.tgz", - "integrity": "sha512-iT40nudw8zmCweivz6j58g+RT33I4KbaIvRUhjNmDwO2WmsQUxFEZZYZ5w3vXe5x5MX9D7mfvA/XaLOZYFR9EQ==", - "requires": { - "core-js": "^2.5.0" - }, - "dependencies": { - "core-js": { - "version": "2.6.10", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.10.tgz", - "integrity": "sha512-I39t74+4t+zau64EN1fE5v2W31Adtc/REhzWN+gWRRXg6WH5qAsZm62DHpQ1+Yhe4047T55jvzz7MUqF/dBBlA==" - } - } + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.0.0.tgz", + "integrity": "sha512-I9SDP4Wvh2ItYYoafEg8hFpsBe96pfQ+eabceShXt3sw2fbIP96+Aoj9zZE0vkZNAkXXzHJATVRuWz+h9FxJxQ==" }, "autoprefixer": { "version": "9.6.1", @@ -3036,6 +3035,31 @@ "triplesec": "^3.0.26", "uuid": "^3.3.2", "zone-file": "^1.0.0" + }, + "dependencies": { + "cheerio": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz", + "integrity": "sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash.assignin": "^4.0.9", + "lodash.bind": "^4.1.4", + "lodash.defaults": "^4.0.1", + "lodash.filter": "^4.4.0", + "lodash.flatten": "^4.2.0", + "lodash.foreach": "^4.3.0", + "lodash.map": "^4.4.0", + "lodash.merge": "^4.4.0", + "lodash.pick": "^4.2.1", + "lodash.reduce": "^4.4.0", + "lodash.reject": "^4.4.0", + "lodash.some": "^4.4.0" + } + } } }, "bluebird": { @@ -3443,6 +3467,22 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "casper": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/casper/-/casper-0.1.1.tgz", + "integrity": "sha1-qerPJphq2twtvHFxtufOtIZnoPc=" + }, + "cfb": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.1.3.tgz", + "integrity": "sha512-joXBW0nMuwV9no7UTMiyVJnQL6XIU3ThXVjFUDHgl9MpILPOomyfaGqC290VELZ48bbQKZXnQ81UT5HouTxHsw==", + "requires": { + "adler-32": "~1.2.0", + "commander": "^2.16.0", + "crc-32": "~1.2.0", + "printj": "~1.1.2" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -3463,29 +3503,6 @@ "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, - "cheerio": { - "version": "0.22.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz", - "integrity": "sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=", - "requires": { - "css-select": "~1.2.0", - "dom-serializer": "~0.1.0", - "entities": "~1.1.1", - "htmlparser2": "^3.9.1", - "lodash.assignin": "^4.0.9", - "lodash.bind": "^4.1.4", - "lodash.defaults": "^4.0.1", - "lodash.filter": "^4.4.0", - "lodash.flatten": "^4.2.0", - "lodash.foreach": "^4.3.0", - "lodash.map": "^4.4.0", - "lodash.merge": "^4.4.0", - "lodash.pick": "^4.2.1", - "lodash.reduce": "^4.4.0", - "lodash.reject": "^4.4.0", - "lodash.some": "^4.4.0" - } - }, "chokidar": { "version": "2.1.8", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", @@ -4224,6 +4241,22 @@ "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, + "codepage": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.14.0.tgz", + "integrity": "sha1-jL4lSBMjVZ19MHVxsP/5HnodL5k=", + "requires": { + "commander": "~2.14.1", + "exit-on-epipe": "~1.0.1" + }, + "dependencies": { + "commander": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.14.1.tgz", + "integrity": "sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw==" + } + } + }, "collection-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", @@ -4532,6 +4565,15 @@ "request": "^2.86.0" } }, + "crc-32": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz", + "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==", + "requires": { + "exit-on-epipe": "~1.0.1", + "printj": "~1.1.0" + } + }, "create-ecdh": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz", @@ -4566,15 +4608,6 @@ "sha.js": "^2.4.8" } }, - "create-react-context": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/create-react-context/-/create-react-context-0.2.3.tgz", - "integrity": "sha512-CQBmD0+QGgTaxDL3OX1IDXYqjkp2It4RIbcb99jS6AEg27Ga+a9G3JtK6SIu0HBwPLZlmwt9F7UwWA4Bn92Rag==", - "requires": { - "fbjs": "^0.8.0", - "gud": "^1.0.0" - } - }, "cross-env": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-6.0.3.tgz", @@ -4959,9 +4992,9 @@ } }, "d3": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/d3/-/d3-5.12.0.tgz", - "integrity": "sha512-flYVMoVuhPFHd9zVCe2BxIszUWqBcd5fvQGMNRmSiBrgdnh6Vlruh60RJQTouAK9xPbOB0plxMvBm4MoyODXNg==", + "version": "5.14.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-5.14.0.tgz", + "integrity": "sha512-RNLVND/lexd7nkTnX75zrRU/wLHMpiW0jo/2GFgx3W4YSCagPvstkQbj+/ziZQZVI5X59qVISGuCE1dpzyS/vw==", "requires": { "d3-array": "1", "d3-axis": "1", @@ -5014,9 +5047,9 @@ "integrity": "sha512-ejINPfPSNdGFKEOAtnBtdkpr24c4d4jsei6Lg98mxf424ivoDP2956/5HDpIAtmHo85lqT4pruy+zEgvRUBqaQ==" }, "d3-brush": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-1.1.3.tgz", - "integrity": "sha512-v8bbYyCFKjyCzFk/tdWqXwDykY8YWqhXYjcYxfILIit085VZOpj4XJKOMccTsvWxgzSLMJQg5SiqHjslsipEDg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-1.1.4.tgz", + "integrity": "sha512-DRcFXGcVZiJF644i78m/HM8P0U19hWYRFcAnacOVGvpOpzkqGm3H0EAoAH6jwyY/lqfH49d+5vz2TgznP/zlMA==", "requires": { "d3-dispatch": "1", "d3-drag": "1", @@ -5067,23 +5100,23 @@ } }, "d3-dispatch": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-1.0.5.tgz", - "integrity": "sha512-vwKx+lAqB1UuCeklr6Jh1bvC4SZgbSqbkGBLClItFBIYH4vqDJCA7qfoy14lXmJdnBOdxndAMxjCbImJYW7e6g==" + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-1.0.6.tgz", + "integrity": "sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==" }, "d3-drag": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-1.2.4.tgz", - "integrity": "sha512-ICPurDETFAelF1CTHdIyiUM4PsyZLaM+7oIBhmyP+cuVjze5vDZ8V//LdOFjg0jGnFIZD/Sfmk0r95PSiu78rw==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-1.2.5.tgz", + "integrity": "sha512-rD1ohlkKQwMZYkQlYVCrSFxsWPzI97+W+PaEIBNTMxRuxz9RF0Hi5nJWHGVJ3Om9d2fRTe1yOBINJyy/ahV95w==", "requires": { "d3-dispatch": "1", "d3-selection": "1" } }, "d3-dsv": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-1.1.1.tgz", - "integrity": "sha512-1EH1oRGSkeDUlDRbhsFytAXU6cAmXFzc52YUe6MRlPClmWb85MP1J5x+YJRzya4ynZWnbELdSAvATFW/MbxaXw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-1.2.0.tgz", + "integrity": "sha512-9yVlqvZcSOMhCYzniHE7EVUws7Fa1zgw+/EAV2BxJoG3ME19V6BQFBwI855XQDsxyOuG7NibqRMTtiF/Qup46g==", "requires": { "commander": "2", "iconv-lite": "0.4", @@ -5091,9 +5124,9 @@ } }, "d3-ease": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-1.0.5.tgz", - "integrity": "sha512-Ct1O//ly5y5lFM9YTdu+ygq7LleSgSE4oj7vUt9tPLHUi8VCV7QoizGpdWRWAwCO9LdYzIrQDg97+hGVdsSGPQ==" + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-1.0.6.tgz", + "integrity": "sha512-SZ/lVU7LRXafqp7XtIcBdxnWl8yyLpgOmzAk0mWBI9gXNzLDx5ybZgnRbH9dN/yY5tzVBqCQ9avltSnqVwessQ==" }, "d3-fetch": { "version": "1.1.2", @@ -5120,9 +5153,9 @@ "integrity": "sha512-TUswGe6hfguUX1CtKxyG2nymO+1lyThbkS1ifLX0Sr+dOQtAD5gkrffpHnx+yHNKUZ0Bmg5T4AjUQwugPDrm0g==" }, "d3-geo": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-1.11.6.tgz", - "integrity": "sha512-z0J8InXR9e9wcgNtmVnPTj0TU8nhYT6lD/ak9may2PdKqXIeHUr8UbFLoCtrPYNsjv6YaLvSDQVl578k6nm7GA==", + "version": "1.11.9", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-1.11.9.tgz", + "integrity": "sha512-9edcH6J3s/Aa3KJITWqFJbyB/8q3mMlA9Fi7z6yy+FAYMnRaxmC7jBhUnsINxVWD14GmqX3DK8uk7nV6/Ekt4A==", "requires": { "d3-array": "1" }, @@ -5135,9 +5168,9 @@ } }, "d3-hierarchy": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.8.tgz", - "integrity": "sha512-L+GHMSZNwTpiq4rt9GEsNcpLa4M96lXMR8M/nMG9p5hBE0jy6C+3hWtyZMenPQdwla249iJy7Nx0uKt3n+u9+w==" + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.9.tgz", + "integrity": "sha512-j8tPxlqh1srJHAtxfvOUwKNYJkQuBFdM1+JAUfq6xqH5eAqf93L7oG1NVqDa4CpFZNvnNKtCYEUC8KY9yEn9lQ==" }, "d3-interpolate": { "version": "1.3.2", @@ -5153,14 +5186,14 @@ "integrity": "sha512-J6EfUNwcMQ+aM5YPOB8ZbgAZu6wc82f/0WFxrxwV6Ll8wBwLaHLKCqQ5Imub02JriCVVdPjgI+6P3a4EWJCxAg==" }, "d3-polygon": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-1.0.5.tgz", - "integrity": "sha512-RHhh1ZUJZfhgoqzWWuRhzQJvO7LavchhitSTHGu9oj6uuLFzYZVeBzaWTQ2qSO6bz2w55RMoOCf0MsLCDB6e0w==" + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-1.0.6.tgz", + "integrity": "sha512-k+RF7WvI08PC8reEoXa/w2nSg5AUMTi+peBD9cmFc+0ixHfbs4QmxxkarVal1IkVkgxVuk9JSHhJURHiyHKAuQ==" }, "d3-quadtree": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-1.0.6.tgz", - "integrity": "sha512-NUgeo9G+ENQCQ1LsRr2qJg3MQ4DJvxcDNCiohdJGHt5gRhBW6orIB5m5FJ9kK3HNL8g9F4ERVoBzcEwQBfXWVA==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-1.0.7.tgz", + "integrity": "sha512-RKPAeXnkC59IDGD0Wu5mANy0Q2V28L+fNe65pOCXVdVuTJS3WPKaJlFHer32Rbh9gIo9qMuJXio8ra4+YmIymA==" }, "d3-random": { "version": "1.1.2", @@ -5206,9 +5239,9 @@ } }, "d3-selection": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.4.0.tgz", - "integrity": "sha512-EYVwBxQGEjLCKF2pJ4+yrErskDnz5v403qvAid96cNdCMr8rmCYfY5RGzWz24mdIbxmDf6/4EAH+K9xperD5jg==" + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-1.4.1.tgz", + "integrity": "sha512-BTIbRjv/m5rcVTfBs4AMBLKs4x8XaaLkwm28KWu9S2vKNqXkXt2AH2Qf0sdPZHjFxcWg/YL53zcqAz+3g4/7PA==" }, "d3-shape": { "version": "1.3.5", @@ -5232,14 +5265,14 @@ } }, "d3-timer": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-1.0.9.tgz", - "integrity": "sha512-rT34J5HnQUHhcLvhSB9GjCkN0Ddd5Y8nCwDBG2u6wQEeYxT/Lf51fTFFkldeib/sE/J0clIe0pnCfs6g/lRbyg==" + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-1.0.10.tgz", + "integrity": "sha512-B1JDm0XDaQC+uvo4DT79H0XmBskgS3l6Ve+1SBCfxgmtIb1AVrPIoqd+nPSv+loMX8szQ0sVUhGngL7D5QPiXw==" }, "d3-transition": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-1.2.0.tgz", - "integrity": "sha512-VJ7cmX/FPIPJYuaL2r1o1EMHLttvoIuZhhuAlRoOxDzogV8iQS6jYulDm3xEU3TqL80IZIhI551/ebmCMrkvhw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-1.3.0.tgz", + "integrity": "sha512-pMwnuuxXct70H8l/nXKmh5lXSesn9kJ0APf4LqrztmWA8Ri4u4fHHZAlIgkvfotCa9FXYlQNvnZfAwF9tuvEKw==", "requires": { "d3-color": "1", "d3-dispatch": "1", @@ -5372,6 +5405,11 @@ "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" }, + "deepmerge": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.2.1.tgz", + "integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA==" + }, "default-gateway": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-4.2.0.tgz", @@ -6652,6 +6690,11 @@ "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=" }, + "exit-on-epipe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz", + "integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==" + }, "expand-brackets": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", @@ -7244,25 +7287,28 @@ } }, "formik": { - "version": "1.5.8", - "resolved": "https://registry.npmjs.org/formik/-/formik-1.5.8.tgz", - "integrity": "sha512-fNvPe+ddbh+7xiByT25vuso2p2hseG/Yvuj211fV1DbCjljUEG9OpgRpcb7g7O3kxHX/q31cbZDzMxJXPWSNwA==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/formik/-/formik-2.0.4.tgz", + "integrity": "sha512-Y0mR8PGRtq/U072U4tkX1wnS/geDYs0n7uPlvmKtMTIJS4g8xCpaccAerQFWxEfClMK/JGpmEyG93zItAdASJA==", "requires": { - "create-react-context": "^0.2.2", "deepmerge": "^2.1.1", "hoist-non-react-statics": "^3.3.0", "lodash": "^4.17.14", "lodash-es": "^4.17.14", - "prop-types": "^15.6.1", "react-fast-compare": "^2.0.1", + "scheduler": "^0.17.0", "tiny-warning": "^1.0.2", - "tslib": "^1.9.3" + "tslib": "^1.10.0" }, "dependencies": { - "deepmerge": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.2.1.tgz", - "integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA==" + "scheduler": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.17.0.tgz", + "integrity": "sha512-7rro8Io3tnCPuY4la/NuI5F2yfESpnfZyT6TtkXnSWVkcu0BCDJ+8gk5ozUaFaxpIyNuWAPXrH0yFcSi28fnDA==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } } } }, @@ -7271,6 +7317,11 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, + "frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==" + }, "fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -13247,6 +13298,11 @@ } } }, + "printj": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz", + "integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==" + }, "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", @@ -13632,12 +13688,12 @@ } }, "react-dropzone": { - "version": "10.1.10", - "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-10.1.10.tgz", - "integrity": "sha512-vcLBdkYo7wgZpw1o4cz7uk8/Mmm+sYHeiTfFSshA/EGthz/TjjrTOrKwvFHm3o1p1LPk+x+KbDDlw5OeIo6eYA==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-10.2.0.tgz", + "integrity": "sha512-VvJtg6GKtM1Xu+SsMcBNBcB2XcOi27xbNLBMDkrpqsk3cSILFiBVoCuW96FSOWkCK1IFeNg67FjKu/c/KuUhkg==", "requires": { - "attr-accept": "^1.1.3", - "file-selector": "^0.1.11", + "attr-accept": "^2.0.0", + "file-selector": "^0.1.12", "prop-types": "^15.7.2" } }, @@ -15302,6 +15358,14 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, + "ssf": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.10.2.tgz", + "integrity": "sha512-rDhAPm9WyIsY8eZEKyE8Qsotb3j/wBdvMWBUsOhJdfhKGLfQidRjiBUV0y/MkyCLiXQ38FG6LWW/VYUtqlIDZQ==", + "requires": { + "frac": "~1.1.2" + } + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -17053,6 +17117,27 @@ "async-limiter": "^1.0.0" } }, + "xlsx": { + "version": "0.15.2", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.15.2.tgz", + "integrity": "sha512-0YcFaYgoqSHh9JcWeQU5X/kR6hyXrs+boc6BomPnJAUVYGgo+Jna9blIwMztjQYfJ6WHs5tKtg5GBBGjkLFVhA==", + "requires": { + "adler-32": "~1.2.0", + "cfb": "^1.1.3", + "codepage": "~1.14.0", + "commander": "~2.17.1", + "crc-32": "~1.2.0", + "exit-on-epipe": "~1.0.1", + "ssf": "~0.10.2" + }, + "dependencies": { + "commander": { + "version": "2.17.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", + "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==" + } + } + }, "xml-name-validator": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", diff --git a/package.json b/package.json index 76fb4f6..d3111ae 100644 --- a/package.json +++ b/package.json @@ -41,13 +41,14 @@ "aos": "^2.3.4", "big.js": "^5.2.2", "blockstack": "^19.2.5", + "casper": "^0.1.1", "chroma-js": "^2.1.0", "classnames": "^2.2.6", - "d3": "^5.12.0", + "d3": "^5.14.0", "d3-array": "^2.4.0", "d3-sankey": "^0.12.3", "date-fns": "^2.7.0", - "formik": "^1.5.8", + "formik": "^2.0.4", "jsdom": "^15.2.1", "localforage": "^1.7.3", "lodash": "^4.17.15", @@ -59,7 +60,7 @@ "react-confirm": "^0.1.18", "react-date-range": "^1.0.0-beta", "react-dom": "^16.12.0", - "react-dropzone": "^10.1.10", + "react-dropzone": "^10.2.0", "react-motion": "^0.5.2", "react-number-format": "^4.3.1", "react-redux": "^7.1.3", @@ -77,6 +78,7 @@ "redux-thunk": "^2.3.0", "reselect": "^4.0.0", "uuid": "^3.3.3", + "xlsx": "^0.15.2", "yup": "^0.27.0" }, "eslintConfig": { diff --git a/src/core/Accounts/Transactions/__tests__/__snapshots__/TransactionDialog.test.jsx.snap b/src/core/Accounts/Transactions/__tests__/__snapshots__/TransactionDialog.test.jsx.snap index 10538eb..aaee819 100644 --- a/src/core/Accounts/Transactions/__tests__/__snapshots__/TransactionDialog.test.jsx.snap +++ b/src/core/Accounts/Transactions/__tests__/__snapshots__/TransactionDialog.test.jsx.snap @@ -10,8 +10,8 @@ exports[`ConfirmDialog matches snapshot 1`] = ` - - + + diff --git a/src/core/BudgetCategories/__tests__/__snapshots__/form.test.jsx.snap b/src/core/BudgetCategories/__tests__/__snapshots__/form.test.jsx.snap index b663ed5..a9f7812 100644 --- a/src/core/BudgetCategories/__tests__/__snapshots__/form.test.jsx.snap +++ b/src/core/BudgetCategories/__tests__/__snapshots__/form.test.jsx.snap @@ -5,8 +5,8 @@ exports[`CategoryForm matches snapshot for existing category 1`] = ` - - + +
@@ -140,8 +140,8 @@ exports[`CategoryForm matches snapshot for new category 1`] = ` - - + + diff --git a/src/core/ImportFromInstitution/__tests__/__snapshots__/form.test.jsx.snap b/src/core/ImportFromInstitution/__tests__/__snapshots__/form.test.jsx.snap index f8ee558..093c8c3 100644 --- a/src/core/ImportFromInstitution/__tests__/__snapshots__/form.test.jsx.snap +++ b/src/core/ImportFromInstitution/__tests__/__snapshots__/form.test.jsx.snap @@ -50,6 +50,9 @@ exports[`Account form matches snapshot for existing Coinbase account 1`] = ` /> +
Import + @@ -284,9 +290,6 @@ exports[`Account form matches snapshot for new Coinbase account 1`] = ` /> -
Import -