Minor fixes and improvements for csv2yaml and transactions scripts

scripts/csv2yaml/dkb-visa.js

@@ -7,6 +7,7 @@ const {
   rmEmptyString,
   keysToEnglish,
   noteToAccount,
+  sanitizeYaml,
 } = require('../helpers.js')
 
 
@@ -74,11 +75,7 @@ function normalizeAndPrint (filePathTemp) {
           .localeCompare(String(transB.utc), 'en'),
       )
 
-    const yamlString = yaml
-      .dump({transactions})
-      .replace(/^ {2}- /gm, '\n  -\n    ')
-      .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2')
-      .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'')
+    const yamlString = sanitizeYaml(yaml.dump({transactions}))
 
     console.info(yamlString)
   })

scripts/csv2yaml/dkb.js

@@ -0,0 +1,91 @@
+const fse = require('fs-extra')
+const yaml = require('js-yaml')
+const csvnorm = require('csvnorm')
+const converter = require('converter')
+
+const {
+  rmEmptyString,
+  keysToEnglish,
+  noteToAccount,
+  sanitizeYaml,
+} = require('../helpers.js')
+
+
+function normalizeAndPrint (filePathTemp) {
+  const csv2json = converter({
+    from: 'csv',
+    to: 'json',
+  })
+
+  let jsonTemp = ''
+  csv2json.on('data', chunk => {
+    jsonTemp += chunk
+  })
+  csv2json.on('end', () => {
+    const transactions = JSON
+      .parse(jsonTemp)
+      .map(keysToEnglish)
+      .map(transaction => {
+        const note = transaction.note
+          .replace(/<br\s+\/>/g, '\n')
+        const amount = transaction.amount + ' €'
+        const sortedTransaction = {
+          utc: transaction['value-utc'] < transaction['entry-utc']
+            ? transaction['value-utc']
+            : transaction['entry-utc'],
+        }
+
+        if (transaction['value-utc'] !== sortedTransaction.utc) {
+          sortedTransaction['value-utc'] = transaction['value-utc']
+        }
+        if (transaction['entry-utc'] !== sortedTransaction.utc) {
+          sortedTransaction['entry-utc'] = transaction['entry-utc']
+        }
+
+        sortedTransaction.type = transaction.type
+        sortedTransaction.note = note
+
+        const transfersObj = transaction.amount.startsWith('-')
+          ? {
+            transfers: [{
+              from: 'dkb:visa',
+              to: noteToAccount(note),
+              amount: amount.slice(1),
+              'original-amount': transaction['original-amount'],
+            }],
+          }
+          : {
+            transfers: [{
+              from: noteToAccount(note),
+              to: 'dkb:visa',
+              // TODO: Remove when github.com/adius/csvnorm/issues/1 is solved
+              amount: transaction.amount === '0,00' ? '0 €' : amount,
+              'original-amount': transaction['original-amount'],
+            }],
+          }
+        const newTransaction = Object.assign(sortedTransaction, transfersObj)
+
+        delete newTransaction.amount
+
+        return JSON.parse(JSON.stringify(newTransaction, rmEmptyString))
+      })
+      .sort((transA, transB) =>
+        // Oldest first
+        String(transA.utc)
+          .localeCompare(String(transB.utc), 'en'),
+      )
+
+    const yamlString = sanitizeYaml(yaml.dump({transactions}))
+
+    console.info(yamlString)
+  })
+
+  csvnorm.default({
+    encoding: 'latin1',
+    readableStream: fse.createReadStream(filePathTemp),
+    skipLinesStart: 6,
+    writableStream: csv2json,
+  })
+}
+
+normalizeAndPrint(process.argv[2])

scripts/csv2yaml/fidor.js

@@ -0,0 +1,74 @@
+const fse = require('fs-extra')
+const yaml = require('js-yaml')
+const csvnorm = require('csvnorm')
+const converter = require('converter')
+
+const {
+  rmEmptyString,
+  keysToEnglish,
+  noteToAccount,
+  sanitizeYaml,
+} = require('../helpers.js')
+
+
+function normalizeAndPrint (filePathTemp) {
+  const csv2json = converter({
+    from: 'csv',
+    to: 'json',
+  })
+
+  let jsonTemp = ''
+  csv2json.on('data', chunk => {
+    jsonTemp += chunk
+  })
+  csv2json.on('end', () => {
+    const transactions = JSON
+      .parse(jsonTemp)
+      .map(keysToEnglish)
+      .reverse() // Now sorted ascending by value date
+      .map(transaction => {
+        const currency = ' €'
+        const sortedTransaction = {
+          utc: transaction.date,
+          note: transaction.note + '\n' + transaction.note2,
+        }
+        const account = noteToAccount(transaction.note) || '_todo_'
+        const transfersObj = transaction.amount.startsWith('-')
+          ? {
+            transfers: [{
+              from: 'fidor:giro',
+              to: account,
+              amount: transaction.amount.slice(1) + currency,
+            }],
+          }
+          : {
+            transfers: [{
+              from: account,
+              to: 'fidor:giro',
+              // TODO: Remove when https://github.com/adius/csvnorm/issues/1
+              //       is solved
+              amount: transaction.amount === '0,00'
+                ? 0
+                : transaction.amount + currency,
+            }],
+          }
+        const newTransaction = Object.assign(sortedTransaction, transfersObj)
+
+        delete newTransaction.amount
+
+        return JSON.parse(JSON.stringify(newTransaction, rmEmptyString))
+      })
+
+    const yamlString = sanitizeYaml(yaml.dump({transactions}))
+
+    console.info(yamlString)
+  })
+
+  csvnorm.default({
+    encoding: 'utf-8',
+    readableStream: fse.createReadStream(filePathTemp),
+    writableStream: csv2json,
+  })
+}
+
+normalizeAndPrint(process.argv[2])

scripts/csv2yaml/finvesto.js

@@ -7,6 +7,7 @@ const {
   rmEmptyString,
   keysToEnglish,
   noteToAccount,
+  sanitizeYaml,
 } = require('../helpers.js')
 
 
@@ -69,11 +70,7 @@ function normalizeAndPrint (filePathTemp) {
         return JSON.parse(JSON.stringify(newTransaction, rmEmptyString))
       })
 
-    const yamlString = yaml
-      .dump({transactions})
-      .replace(/^ {2}- /gm, '\n {2}-\n {4}')
-      .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2')
-      .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'')
+    const yamlString = sanitizeYaml(yaml.dump({transactions}))
 
     console.info(yamlString)
   })

scripts/csv2yaml/hypovereinsbank.js

@@ -7,6 +7,7 @@ const {
   rmEmptyString,
   keysToEnglish,
   noteToAccount,
+  sanitizeYaml,
 } = require('../helpers.js')
 
 
@@ -66,11 +67,7 @@ function normalizeAndPrint (filePathTemp) {
         return JSON.parse(JSON.stringify(newTransaction, rmEmptyString))
       })
 
-    const yamlString = yaml
-      .dump({transactions})
-      .replace(/^ {2}- /gm, '\n  -\n    ')
-      .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2')
-      .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'')
+    const yamlString = sanitizeYaml(yaml.dump({transactions}))
 
     console.info(yamlString)
   })

scripts/csv2yaml/mbs.js

@@ -7,6 +7,7 @@ const {
   rmEmptyString,
   keysToEnglish,
   noteToAccount,
+  sanitizeYaml,
 } = require('../helpers.js')
 
 
@@ -41,7 +42,7 @@ function normalizeAndPrint (filePathTemp) {
           delete sortedTransaction['entry-utc']
         }
 
-        const account = noteToAccount(transaction.to)
+        const account = noteToAccount(transaction.to) || '_todo_'
         const transfersObj = transaction.amount.startsWith('-')
           ? {
             transfers: [{
@@ -69,11 +70,7 @@ function normalizeAndPrint (filePathTemp) {
         return JSON.parse(JSON.stringify(newTransaction, rmEmptyString))
       })
 
-    const yamlString = yaml
-      .dump({transactions})
-      .replace(/^ {2}- /gm, '\n  -\n    ')
-      .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2')
-      .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'')
+    const yamlString = sanitizeYaml(yaml.dump({transactions}))
 
     console.info(yamlString)
   })

scripts/csv2yaml/paypal.js

@@ -8,6 +8,8 @@ const chrono = require('chrono-node')
 const {
   rmEmptyString,
   keysToEnglish,
+  noteToAccount,
+  sanitizeYaml,
 } = require('../helpers.js')
 
 
@@ -46,21 +48,18 @@ function normalizeAndPrint (filePathTemp) {
           },
           transaction,
         )
+        const account = noteToAccount(transaction.Name)
         const transfer = transaction.Gross.startsWith('-')
           ? {
             from: '_todo_:paypal:' +
               transaction.Currency
                 .toLowerCase()
                 .trim(),
-            to: transaction.Name
-              ? `${transaction.Name} <${transaction['To Email Address']}>`
-              : 'paypal',
+            to: account ? account : 'paypal',
             amount: transaction.Gross.slice(1) + ' ' + currency,
           }
           : {
-            from: transaction.Name
-              ? `${transaction.Name} <${transaction['From Email Address']}>`
-              : 'paypal',
+            from: account ? account : 'paypal',
             to: '_todo_:paypal:' +
               transaction.Currency
                 .toLowerCase()
@@ -101,11 +100,7 @@ function normalizeAndPrint (filePathTemp) {
         return JSON.parse(JSON.stringify(newTransaction, rmEmptyString))
       })
 
-    const yamlString = yaml
-      .dump({transactions})
-      .replace(/^ {2}- /gm, '\n  -\n    ')
-      .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2')
-      .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'')
+    const yamlString = sanitizeYaml(yaml.dump({transactions}))
 
     console.info(yamlString)
   })

scripts/helpers.js

@@ -4,8 +4,20 @@ module.exports = {
   prettyFormat,
   prettyPrint,
   rmEmptyString,
-  toDdotMdotYYYY,
+  sanitizeYaml,
   toDDdotMMdotYYYY,
+  toDdotMdotYYYY,
+}
+
+
+function sanitizeYaml (yaml) {
+  if (typeof yaml !== 'string') {
+    throw new Error('YAML must be passed as a string')
+  }
+  return yaml
+    .replace(/^ {2}- /gm, '\n  -\n    ')
+    .replace(/^ {2}([\w- ]+): '(.+)'$/gm, '  $1: $2')
+    .replace(/utc: ([0-9TZ:.-]+)$/gm, 'utc: \'$1\'')
 }
 
 
@@ -80,6 +92,7 @@ function keysToEnglish (object) {
         .replace('Verwendungszweck', 'note')
         .replace('Waehrung', 'currency')
         .replace('Wertstellung', 'value-utc')
+        .replace('Wert', 'amount')
         .replace('Zahlungsbetrag in ZW', 'amount')
         .replace('Zahlungswährung (ZW)', 'currency')
 
@@ -89,6 +102,9 @@ function keysToEnglish (object) {
 }
 
 function noteToAccount (note) {
+  // Remove misleading terms
+  note = note.replace('Apple Pay', '')
+
   // Sorted by ascending importance
   // I.e. later keywords overwrite selection
   /* eslint-disable quote-props */
@@ -98,11 +114,13 @@ function noteToAccount (note) {
     'amazon': 'amazon',
     'amazon prime': 'amazon:prime',
     'patreon': 'patreon',
+    'facebook': 'facebook',
     'google ireland limited cloud platform': 'google:cloud',
     'day night sports gmbh': 'day_night_sports',
     'spotify': 'spotify',
     'apple': 'apple',
     'itunes': 'apple:itunes',
+    'mozilla foundation': 'mozilla_foundation',
     'mozilla': 'mozilla',
     'namecheap': 'namecheap',
     'name-cheap': 'namecheap',
@@ -130,6 +148,14 @@ function noteToAccount (note) {
     'jimmy joy': 'jimmy_joy',
     'wecircberlin': 'circ',
     'lime ride': 'lime',
+    'lime electric': 'lime',
+    'bird rides': 'bird',
+    'google': 'google',
+    'free software foundation': 'free_software_foundation',
+    'landr audio': 'landr',
+    'mzla technologies': 'mzla_technologies',
+    'vodafone': 'vodafone',
+    'mailgun': 'mailgun',
 
     // German
     'ihk ': 'ihk',
@@ -159,11 +185,15 @@ function noteToAccount (note) {
     'qthority': 'qthority',
     'nosh good taste': 'nosh',
     'musikhaus thomann': 'thomann',
-    'fratellis frankfurt': 'fratellis_frankfurt',
+    'fratellis frankfurt': 'fratellis_ristorante',
+    'fratellis rist': 'fratellis_ristorante',
     'hansemerkur speziale kv': 'hansemerkur',
     'zwanzigeins e.v.': 'zwanzigeins',
     'tier de': 'tier',
     'zalando': 'zalando',
+    'hetzner online': 'hetzner',
+    'golem media': 'golem',
+    'pro rauchfrei': 'pro_rauchfrei',
   }
   /* eslint-enable quote-props */
   let account = note
@@ -180,5 +210,5 @@ function noteToAccount (note) {
       }
     })
 
-  return account || '_todo_'
+  return account
 }