diff --git a/beancount_import/source/generic_importer_source.py b/beancount_import/source/generic_importer_source.py index 76c6427d..d7bde273 100644 --- a/beancount_import/source/generic_importer_source.py +++ b/beancount_import/source/generic_importer_source.py @@ -39,19 +39,19 @@ def __init__(self, directory: str, account: str, importer: ImporterProtocol, - account_name:str = None, **kwargs) -> None: super().__init__(**kwargs) self.directory = os.path.expanduser(directory) self.importer = importer self.account = account - self.account_name = account_name if account_name else self.name self._comparator = SimilarityComparator() # get _FileMemo object for each file files = [get_file(f) for f in + filter(os.path.isfile, glob(os.path.join(directory, '**', '*'), recursive=True) + ) ] # filter the valid files for this importer self.files = [f for f in files if self.importer.identify(f)] @@ -64,7 +64,7 @@ def prepare(self, journal: 'JournalEditor', results: SourceResults) -> None: results.add_account(self.account) entries = defaultdict(list) for f in self.files: - f_entries = self.importer.extract(f) + f_entries = self.importer.extract(f, existing_entries=journal.entries) # collect all entries in current statement, grouped by hash hashed_entries = defaultdict(list) for entry in f_entries: @@ -109,6 +109,7 @@ def _add_description(self, entry: Transaction): postings = entry.postings #type: ignore to_mutate = [] for i, posting in enumerate(postings): + if posting.account != self.account: continue if isinstance(posting.meta, dict): posting.meta["source_desc"] = entry.narration else: to_mutate.append(i) for i in to_mutate: @@ -182,5 +183,6 @@ def similar_entries_in_journal(entry:Transaction, source_entries:List[Directive] duplicates.append(source_entry) return duplicates + def load(spec, log_status): return ImporterSource(log_status=log_status, **spec) diff --git a/beancount_import/source/generic_importer_source_test.py b/beancount_import/source/generic_importer_source_test.py new file mode 100644 index 00000000..0ddc912d --- /dev/null +++ b/beancount_import/source/generic_importer_source_test.py @@ -0,0 +1,37 @@ +import os + +import pytest + +from .source_test import check_source_example +from beancount.ingest.importers.csv import Importer as CSVImporter, Col + +testdata_dir = os.path.realpath( + os.path.join( + os.path.dirname(__file__), '..', '..', 'testdata', 'source', 'generic_importer')) + +examples = [ + 'test_basic', + 'test_training_examples' +] + +importer = CSVImporter({Col.DATE: 'Date', + Col.NARRATION1: 'Description', + Col.AMOUNT: 'Amount', + }, + 'Assets:Bank', + 'USD', + '"Date","Description","Amount"', + ) + + +@pytest.mark.parametrize('name', examples) +def test_source(name: str): + check_source_example( + example_dir=os.path.join(testdata_dir, name), + source_spec={ + 'module': 'beancount_import.source.generic_importer_source', + 'directory': testdata_dir, + 'account': 'Assets:Bank', + 'importer': importer, + }, + replacements=[(testdata_dir, '')]) diff --git a/testdata/source/generic_importer/generic_statement.csv b/testdata/source/generic_importer/generic_statement.csv new file mode 100644 index 00000000..5d46d425 --- /dev/null +++ b/testdata/source/generic_importer/generic_statement.csv @@ -0,0 +1,7 @@ +"Date","Description","Amount" +2020-01-01,by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-,-1 +2020-01-01,by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-,-1 +2020-01-02,BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-,1 +2020-01-02,ATM-WD Some Random ATM Machine,500 +2020-01-02,BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-,1 +2020-01-05,Transfer to 1234567890123,300 diff --git a/testdata/source/generic_importer/test_basic/accounts.txt b/testdata/source/generic_importer/test_basic/accounts.txt new file mode 100644 index 00000000..255958da --- /dev/null +++ b/testdata/source/generic_importer/test_basic/accounts.txt @@ -0,0 +1 @@ +Assets:Bank diff --git a/testdata/source/generic_importer/test_basic/import_results.beancount b/testdata/source/generic_importer/test_basic/import_results.beancount new file mode 100644 index 00000000..d1f407eb --- /dev/null +++ b/testdata/source/generic_importer/test_basic/import_results.beancount @@ -0,0 +1,107 @@ +;; date: 2020-01-01 +;; info: null + +; features: [ +; { +; "amount": "-1 USD", +; "date": "2020-01-01", +; "key_value_pairs": { +; "desc": "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-" +; }, +; "source_account": "Assets:Bank" +; } +; ] +2020-01-01 * "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-" + Assets:Bank -1 USD + source_desc: "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-" + Expenses:FIXME 1 USD + +;; date: 2020-01-01 +;; info: null + +; features: [ +; { +; "amount": "-1 USD", +; "date": "2020-01-01", +; "key_value_pairs": { +; "desc": "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-" +; }, +; "source_account": "Assets:Bank" +; } +; ] +2020-01-01 * "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-" + Assets:Bank -1 USD + source_desc: "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-" + Expenses:FIXME 1 USD + +;; date: 2020-01-02 +;; info: null + +; features: [ +; { +; "amount": "1 USD", +; "date": "2020-01-02", +; "key_value_pairs": { +; "desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" +; }, +; "source_account": "Assets:Bank" +; } +; ] +2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + Assets:Bank 1 USD + source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + Expenses:FIXME -1 USD + +;; date: 2020-01-02 +;; info: null + +; features: [ +; { +; "amount": "1 USD", +; "date": "2020-01-02", +; "key_value_pairs": { +; "desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" +; }, +; "source_account": "Assets:Bank" +; } +; ] +2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + Assets:Bank 1 USD + source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + Expenses:FIXME -1 USD + +;; date: 2020-01-02 +;; info: null + +; features: [ +; { +; "amount": "500 USD", +; "date": "2020-01-02", +; "key_value_pairs": { +; "desc": "ATM-WD Some Random ATM Machine" +; }, +; "source_account": "Assets:Bank" +; } +; ] +2020-01-02 * "ATM-WD Some Random ATM Machine" + Assets:Bank 500 USD + source_desc: "ATM-WD Some Random ATM Machine" + Expenses:FIXME -500 USD + +;; date: 2020-01-05 +;; info: null + +; features: [ +; { +; "amount": "300 USD", +; "date": "2020-01-05", +; "key_value_pairs": { +; "desc": "Transfer to 1234567890123" +; }, +; "source_account": "Assets:Bank" +; } +; ] +2020-01-05 * "Transfer to 1234567890123" + Assets:Bank 300 USD + source_desc: "Transfer to 1234567890123" + Expenses:FIXME -300 USD diff --git a/testdata/source/generic_importer/test_basic/journal.beancount b/testdata/source/generic_importer/test_basic/journal.beancount new file mode 100644 index 00000000..431f1819 --- /dev/null +++ b/testdata/source/generic_importer/test_basic/journal.beancount @@ -0,0 +1 @@ +1900-01-01 open Assets:Bank diff --git a/testdata/source/generic_importer/test_basic/training_examples.json b/testdata/source/generic_importer/test_basic/training_examples.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/testdata/source/generic_importer/test_basic/training_examples.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/testdata/source/generic_importer/test_training_examples/accounts.txt b/testdata/source/generic_importer/test_training_examples/accounts.txt new file mode 100644 index 00000000..255958da --- /dev/null +++ b/testdata/source/generic_importer/test_training_examples/accounts.txt @@ -0,0 +1 @@ +Assets:Bank diff --git a/testdata/source/generic_importer/test_training_examples/import_results.beancount b/testdata/source/generic_importer/test_training_examples/import_results.beancount new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/testdata/source/generic_importer/test_training_examples/import_results.beancount @@ -0,0 +1 @@ + diff --git a/testdata/source/generic_importer/test_training_examples/journal.beancount b/testdata/source/generic_importer/test_training_examples/journal.beancount new file mode 100644 index 00000000..c31042f0 --- /dev/null +++ b/testdata/source/generic_importer/test_training_examples/journal.beancount @@ -0,0 +1,41 @@ +1900-01-01 open Assets:Bank +1900-01-01 open Assets:Cash +1900-01-01 open Expenses:Misc +1900-01-01 open Liabilities:JohnDoe + + +2020-01-01 * "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-" + Assets:Bank -1 USD + source_desc: "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-" + cleared: TRUE + Expenses:Misc 1 USD + +2020-01-01 * "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-" + Assets:Bank -1 USD + source_desc: "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-" + cleared: TRUE + Expenses:Misc 1 USD + +2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + Assets:Bank 1 USD + source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + cleared: TRUE + Expenses:Misc -1 USD + +2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + Assets:Bank 1 USD + source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + cleared: TRUE + Expenses:Misc -1 USD + +2020-01-02 * "ATM-WD Some Random ATM Machine" + Assets:Bank 500 USD + source_desc: "ATM-WD Some Random ATM Machine" + cleared: TRUE + Assets:Cash -500 USD + +2020-01-05 * "Transfer to 1234567890123" + Assets:Bank 300 USD + source_desc: "Transfer to 1234567890123" + cleared: TRUE + Liabilities:JohnDoe -300 USD diff --git a/testdata/source/generic_importer/test_training_examples/training_examples.json b/testdata/source/generic_importer/test_training_examples/training_examples.json new file mode 100644 index 00000000..cae7eff6 --- /dev/null +++ b/testdata/source/generic_importer/test_training_examples/training_examples.json @@ -0,0 +1,68 @@ +[ + [ + { + "amount": "-1 USD", + "date": "2020-01-01", + "key_value_pairs": { + "desc": "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-" + }, + "source_account": "Assets:Bank" + }, + "Expenses:Misc" + ], + [ + { + "amount": "-1 USD", + "date": "2020-01-01", + "key_value_pairs": { + "desc": "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-" + }, + "source_account": "Assets:Bank" + }, + "Expenses:Misc" + ], + [ + { + "amount": "1 USD", + "date": "2020-01-02", + "key_value_pairs": { + "desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + }, + "source_account": "Assets:Bank" + }, + "Expenses:Misc" + ], + [ + { + "amount": "1 USD", + "date": "2020-01-02", + "key_value_pairs": { + "desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-" + }, + "source_account": "Assets:Bank" + }, + "Expenses:Misc" + ], + [ + { + "amount": "500 USD", + "date": "2020-01-02", + "key_value_pairs": { + "desc": "ATM-WD Some Random ATM Machine" + }, + "source_account": "Assets:Bank" + }, + "Assets:Cash" + ], + [ + { + "amount": "300 USD", + "date": "2020-01-05", + "key_value_pairs": { + "desc": "Transfer to 1234567890123" + }, + "source_account": "Assets:Bank" + }, + "Liabilities:JohnDoe" + ] +] \ No newline at end of file