Merge pull request #66 from dumbPy/documentation/importer-source

Add ImporterSource to Readme and examples
jbms · Dec 6, 2020 · b3b4814 · b3b4814
2 parents 4ef7aa3 + a04615e
commit b3b4814
Show file tree

Hide file tree

Showing 10 changed files with 156 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -12,6 +12,9 @@ each other and with existing transactions.
 - Pluggable data source architecture, including existing support for OFX (cash,
   investment, and retirement accounts), Mint.com, Amazon.com, and Venmo.
 
+- Supports [beancount importers](https://beancount.github.io/docs/importing_external_data.html) so it's easier to write your own,
+  and existing beancount and fava users can hop right on with no hustle.
+
 - Robustly associates imported transactions with the source data, to
   automatically avoid duplicates.
 
@@ -124,6 +127,8 @@ The currently supported set of data sources is:
   [Morgan Stanley StockPlan Connect](https://stockplanconnect.com).
 - [beancount_import.source.ultipro_google](beancount_import/source/ultipro_google.py):
   imports Google employee Ultipro payroll statements.
+- [beancount_import.source.generic_importer_source](beancount_import/source/generic_importer_source.py):
+  imports from `beancount.ingest.importer.ImporterProtocol` subclass Importers. See [beancount's documentation](https://beancount.github.io/docs/importing_external_data.html#the-importing-process) on how to write one and checkout the [examples](examples/) directory for a simple csv importer
 
 Refer to the individual data source documentation for details on configuration.
 

diff --git a/examples/data/importers/bank.csv b/examples/data/importers/bank.csv
@@ -0,0 +1,8 @@
+"Date","Description","Amount"
+2020-01-01,by debit card-OTHPG 063441 GOOGLE CLOUD,-1
+2020-01-01,by debit card-OTHPG 063444 GOOGLE CLOUD,-1
+2020-01-02,BULK POSTING- 00000008237 250120 GOOGLE,1
+2020-01-02,ATM-WD Some Random ATM Machine,-500
+2020-01-02,BULK POSTING- 00000008237 250120 GOOGLE,1
+2020-01-05,Transfer to 1234567890123,300
+2020-01-14,Transfer to Amex 431145642232,-30
diff --git a/examples/data/importers/creditcard.csv b/examples/data/importers/creditcard.csv
@@ -0,0 +1,4 @@
+Credit Card Number: 1234XXXXXXXX7890
+Date,Description,Amount,Balance
+2020-01-10,Amazon Order 54429302,-15.5,-35.5
+2020-01-15,Payment Received,30,-5.5
diff --git a/examples/fresh/_config.py b/examples/fresh/_config.py
@@ -0,0 +1,45 @@
+"""
+This config is where you would initialize your importers with personal info
+like account number or credit card last4 digit.
+
+you may also define CONFIG:List[ImporterProtocol] for other beancount tools like
+bean-identify, bean-file, and other beancount scripts to use
+eg. `bean-identify _config.py ~/Downloads`
+to identify the files that importers defined here can process
+
+
+beancount-import should have it's own run.py where you invoke the
+`beancount_import.webserver.main` but import the Importer objects from this config
+
+This is the way!!
+"""
+from beancount.ingest.importers.csv import Importer as CSVImporter, Col
+
+my_foobar_bank_importer = CSVImporter({
+                        Col.DATE: 'Date',
+                        Col.NARRATION1: 'Description',
+                        Col.AMOUNT: 'Amount',
+                        },
+                       'Assets:FooBarBank', # account
+                       'EUR', # currency
+                        # regexps used by ImporterProtocol.identify() to identify the correct file
+                       '"Date","Description","Amount"',
+                       )
+
+my_amex_cc_importer = CSVImporter({
+                        Col.DATE: 'Date',
+                        Col.NARRATION1: 'Description',
+                        Col.AMOUNT: 'Amount',
+                        Col.BALANCE:'Balance'
+                        },
+                       'Liabilities:Amex-Credit-Card', # account
+                       'EUR', # currency
+                        # regexps used by ImporterProtocol.identify() to identify the correct file
+                       ('Date,Description,Amount,Balance',
+                       'Credit.*7890'
+                       ),
+                       skip_lines=1
+                       )
+
+# beancount's scripts use this
+CONFIG = [my_foobar_bank_importer, my_amex_cc_importer]
diff --git a/examples/fresh/accounts.beancount b/examples/fresh/accounts.beancount
@@ -23,3 +23,7 @@
 
 1900-01-01 open Liabilities:Amazon-Store-Card  USD
   mint_id: "Amazon Store Card"
+
+1900-01-01 open Assets:FooBarBank EUR
+
+1900-01-01 open Liabilities:Amex-Credit-Card EUR
diff --git a/examples/fresh/run.py b/examples/fresh/run.py
@@ -5,6 +5,7 @@
 import json
 import sys
 
+from _config import my_foobar_bank_importer, my_amex_cc_importer
 
 def run_reconcile(extra_args):
     import beancount_import.webserver
@@ -32,6 +33,18 @@ def run_reconcile(extra_args):
                 rewards_points='Income:Amazon:Cashback',
             ),
         ),
+        dict(
+            module='beancount_import.source.generic_importer_source',
+            importer=my_foobar_bank_importer,
+            account='Assets:FooBarBank',
+            directory=os.path.join(data_dir, 'importers')
+        ),
+        dict(
+            module='beancount_import.source.generic_importer_source',
+            importer=my_amex_cc_importer,
+            account='Liabilities:Amex-Credit-Card',
+            directory=os.path.join(data_dir, 'importers')
+        ),
     ]
 
     beancount_import.webserver.main(

diff --git a/examples/manually_entered/_config.py b/examples/manually_entered/_config.py
@@ -0,0 +1,44 @@
+"""
+This config is where you would initialize your importers with personal info
+like account number or credit card last4 digit.
+
+you may also define CONFIG:List[ImporterProtocol] for other beancount tools like
+bean-identify, bean-file, and other beancount scripts to use
+eg. `bean-identify _config.py ~/Downloads`
+to identify the files that importers defined here can process
+
+
+beancount-import should have it's own run.py where you invoke the
+`beancount_import.webserver.main` but import the Importer objects from this config
+
+This is the way!!
+"""
+from beancount.ingest.importers.csv import Importer as CSVImporter, Col
+
+my_foobar_bank_importer = CSVImporter({
+                        Col.DATE: 'Date',
+                        Col.NARRATION1: 'Description',
+                        Col.AMOUNT: 'Amount',
+                        },
+                       'Assets:FooBarBank', # account
+                       'EUR', # currency
+                        # regexps used by ImporterProtocol.identify() to identify the correct file
+                       '"Date","Description","Amount"',
+                       )
+
+my_amex_cc_importer = CSVImporter({
+                        Col.DATE: 'Date',
+                        Col.NARRATION1: 'Description',
+                        Col.AMOUNT: 'Amount',
+                        Col.BALANCE:'Balance'
+                        },
+                       'Liabilities:Amex-Credit-Card', # account
+                       'EUR', # currency
+                        # regexps used by ImporterProtocol.identify() to identify the correct file
+                       ('Date,Description,Amount,Balance',
+                       'Credit.*7890'
+                       ),
+                       skip_lines=1
+                       )
+# beancount's scripts use this
+CONFIG = [my_foobar_bank_importer, my_amex_cc_importer]
diff --git a/examples/manually_entered/accounts.beancount b/examples/manually_entered/accounts.beancount
@@ -38,3 +38,11 @@
 2010-06-13 open Liabilities:Credit-Card:Wells-Fargo         USD
 
 2018-06-14 open Income:My-Company:Salary                        USD
+
+1994-01-01 open Expenses:ImaginarySon:Education EUR
+
+1900-01-01 open Assets:FooBarBank EUR
+
+1900-01-01 open Expenses:Groceries
+
+1900-01-01 open Liabilities:Amex-Credit-Card EUR
diff --git a/examples/manually_entered/run.py b/examples/manually_entered/run.py
@@ -5,6 +5,7 @@
 import json
 import sys
 
+from _config import my_foobar_bank_importer, my_amex_cc_importer
 
 def run_reconcile(extra_args):
     import beancount_import.webserver
@@ -22,6 +23,18 @@ def run_reconcile(extra_args):
             ofx_filenames=[os.path.join(data_dir, 'ofx/checking2.ofx'),
             ],
         ),
+        dict(
+            module='beancount_import.source.generic_importer_source',
+            importer=my_foobar_bank_importer,
+            account='Assets:FooBarBank',
+            directory=os.path.join(data_dir, 'importers')
+        ),
+        dict(
+            module='beancount_import.source.generic_importer_source',
+            importer=my_amex_cc_importer,
+            account='Liabilities:Amex-Credit-Card',
+            directory=os.path.join(data_dir, 'importers')
+        ),
     ]
 
     beancount_import.webserver.main(

diff --git a/examples/manually_entered/transactions.beancount b/examples/manually_entered/transactions.beancount
@@ -57,3 +57,15 @@
 2018-06-15 * "Coffee"
   Assets:Checking:Chase  -70.71 USD
   Expenses:Coffee         70.71 USD
+
+2020-01-02 * "Imaginary Son" "Withdrawn 500 EURs for my Imaginary Son's College Fees"
+  Assets:FooBarBank -500 EUR
+  Expenses:ImaginarySon:Education
+
+2020-01-10 * "Groceries from Amazon"
+  Liabilities:Amex-Credit-Card                -15.5 EUR
+  Expenses:Groceries
+
+2020-01-14 * "Paid 30 EUR to Amex Credit Card"
+  Assets:FooBarBank -30 EUR
+  Liabilities:Amex-Credit-Card