Skip to content

Commit

Permalink
Add tests
Browse files Browse the repository at this point in the history 
and some  very minor changes in ImporterSource
  • Loading branch information
@dumbPy
dumbPy committed Aug 6, 2020
1 parent ca47baa commit bea4b47
Show file tree
Hide file tree
Showing 11 changed files with 270 additions and 3 deletions.
8 changes: 5 additions & 3 deletions beancount_import/source/generic_importer_source.py
View file
Expand Up @@ -39,19 +39,19 @@ def __init__(self,
directory: str,
account: str,
importer: ImporterProtocol,
account_name:str = None,
**kwargs) -> None:
super().__init__(**kwargs)
self.directory = os.path.expanduser(directory)
self.importer = importer
self.account = account
self.account_name = account_name if account_name else self.name

self._comparator = SimilarityComparator()

# get _FileMemo object for each file
files = [get_file(f) for f in
filter(os.path.isfile,
glob(os.path.join(directory, '**', '*'), recursive=True)
)
]
# filter the valid files for this importer
self.files = [f for f in files if self.importer.identify(f)]
Expand All @@ -64,7 +64,7 @@ def prepare(self, journal: 'JournalEditor', results: SourceResults) -> None:
results.add_account(self.account)
entries = defaultdict(list)
for f in self.files:
f_entries = self.importer.extract(f)
f_entries = self.importer.extract(f, existing_entries=journal.entries)
# collect all entries in current statement, grouped by hash
hashed_entries = defaultdict(list)
for entry in f_entries:
Expand Down Expand Up @@ -109,6 +109,7 @@ def _add_description(self, entry: Transaction):
postings = entry.postings #type: ignore
to_mutate = []
for i, posting in enumerate(postings):
if posting.account != self.account: continue
if isinstance(posting.meta, dict): posting.meta["source_desc"] = entry.narration
else: to_mutate.append(i)
for i in to_mutate:
Expand Down Expand Up @@ -182,5 +183,6 @@ def similar_entries_in_journal(entry:Transaction, source_entries:List[Directive]
duplicates.append(source_entry)
return duplicates


def load(spec, log_status):
return ImporterSource(log_status=log_status, **spec)
37 changes: 37 additions & 0 deletions beancount_import/source/generic_importer_source_test.py
View file
@@ -0,0 +1,37 @@
import os

import pytest

from .source_test import check_source_example
from beancount.ingest.importers.csv import Importer as CSVImporter, Col

testdata_dir = os.path.realpath(
os.path.join(
os.path.dirname(__file__), '..', '..', 'testdata', 'source', 'generic_importer'))

examples = [
'test_basic',
'test_training_examples'
]

importer = CSVImporter({Col.DATE: 'Date',
Col.NARRATION1: 'Description',
Col.AMOUNT: 'Amount',
},
'Assets:Bank',
'USD',
'"Date","Description","Amount"',
)


@pytest.mark.parametrize('name', examples)
def test_source(name: str):
check_source_example(
example_dir=os.path.join(testdata_dir, name),
source_spec={
'module': 'beancount_import.source.generic_importer_source',
'directory': testdata_dir,
'account': 'Assets:Bank',
'importer': importer,
},
replacements=[(testdata_dir, '<testdata>')])
7 changes: 7 additions & 0 deletions testdata/source/generic_importer/generic_statement.csv
View file
@@ -0,0 +1,7 @@
"Date","Description","Amount"
2020-01-01,by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-,-1
2020-01-01,by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-,-1
2020-01-02,BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-,1
2020-01-02,ATM-WD Some Random ATM Machine,500
2020-01-02,BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-,1
2020-01-05,Transfer to 1234567890123,300
1 change: 1 addition & 0 deletions testdata/source/generic_importer/test_basic/accounts.txt
View file
@@ -0,0 +1 @@
Assets:Bank
107 changes: 107 additions & 0 deletions testdata/source/generic_importer/test_basic/import_results.beancount
View file
@@ -0,0 +1,107 @@
;; date: 2020-01-01
;; info: null

; features: [
; {
; "amount": "-1 USD",
; "date": "2020-01-01",
; "key_value_pairs": {
; "desc": "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-"
; },
; "source_account": "Assets:Bank"
; }
; ]
2020-01-01 * "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-"
Assets:Bank -1 USD
source_desc: "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-"
Expenses:FIXME 1 USD

;; date: 2020-01-01
;; info: null

; features: [
; {
; "amount": "-1 USD",
; "date": "2020-01-01",
; "key_value_pairs": {
; "desc": "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-"
; },
; "source_account": "Assets:Bank"
; }
; ]
2020-01-01 * "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-"
Assets:Bank -1 USD
source_desc: "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-"
Expenses:FIXME 1 USD

;; date: 2020-01-02
;; info: null

; features: [
; {
; "amount": "1 USD",
; "date": "2020-01-02",
; "key_value_pairs": {
; "desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
; },
; "source_account": "Assets:Bank"
; }
; ]
2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
Assets:Bank 1 USD
source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
Expenses:FIXME -1 USD

;; date: 2020-01-02
;; info: null

; features: [
; {
; "amount": "1 USD",
; "date": "2020-01-02",
; "key_value_pairs": {
; "desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
; },
; "source_account": "Assets:Bank"
; }
; ]
2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
Assets:Bank 1 USD
source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
Expenses:FIXME -1 USD

;; date: 2020-01-02
;; info: null

; features: [
; {
; "amount": "500 USD",
; "date": "2020-01-02",
; "key_value_pairs": {
; "desc": "ATM-WD Some Random ATM Machine"
; },
; "source_account": "Assets:Bank"
; }
; ]
2020-01-02 * "ATM-WD Some Random ATM Machine"
Assets:Bank 500 USD
source_desc: "ATM-WD Some Random ATM Machine"
Expenses:FIXME -500 USD

;; date: 2020-01-05
;; info: null

; features: [
; {
; "amount": "300 USD",
; "date": "2020-01-05",
; "key_value_pairs": {
; "desc": "Transfer to 1234567890123"
; },
; "source_account": "Assets:Bank"
; }
; ]
2020-01-05 * "Transfer to 1234567890123"
Assets:Bank 300 USD
source_desc: "Transfer to 1234567890123"
Expenses:FIXME -300 USD
1 change: 1 addition & 0 deletions testdata/source/generic_importer/test_basic/journal.beancount
View file
@@ -0,0 +1 @@
1900-01-01 open Assets:Bank
1 change: 1 addition & 0 deletions testdata/source/generic_importer/test_basic/training_examples.json
View file
@@ -0,0 +1 @@
[]
1 change: 1 addition & 0 deletions testdata/source/generic_importer/test_training_examples/accounts.txt
View file
@@ -0,0 +1 @@
Assets:Bank
1 change: 1 addition & 0 deletions testdata/source/generic_importer/test_training_examples/import_results.beancount
View file
@@ -0,0 +1 @@

41 changes: 41 additions & 0 deletions testdata/source/generic_importer/test_training_examples/journal.beancount
View file
@@ -0,0 +1,41 @@
1900-01-01 open Assets:Bank
1900-01-01 open Assets:Cash
1900-01-01 open Expenses:Misc
1900-01-01 open Liabilities:JohnDoe


2020-01-01 * "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-"
Assets:Bank -1 USD
source_desc: "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-"
cleared: TRUE
Expenses:Misc 1 USD

2020-01-01 * "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-"
Assets:Bank -1 USD
source_desc: "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-"
cleared: TRUE
Expenses:Misc 1 USD

2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
Assets:Bank 1 USD
source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
cleared: TRUE
Expenses:Misc -1 USD

2020-01-02 * "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
Assets:Bank 1 USD
source_desc: "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
cleared: TRUE
Expenses:Misc -1 USD

2020-01-02 * "ATM-WD Some Random ATM Machine"
Assets:Bank 500 USD
source_desc: "ATM-WD Some Random ATM Machine"
cleared: TRUE
Assets:Cash -500 USD

2020-01-05 * "Transfer to 1234567890123"
Assets:Bank 300 USD
source_desc: "Transfer to 1234567890123"
cleared: TRUE
Liabilities:JohnDoe -300 USD
68 changes: 68 additions & 0 deletions testdata/source/generic_importer/test_training_examples/training_examples.json
View file
@@ -0,0 +1,68 @@
[
[
{
"amount": "-1 USD",
"date": "2020-01-01",
"key_value_pairs": {
"desc": "by debit card-OTHPG 063441 GOOGLE CLOUD INDIA PVTTHANE-"
},
"source_account": "Assets:Bank"
},
"Expenses:Misc"
],
[
{
"amount": "-1 USD",
"date": "2020-01-01",
"key_value_pairs": {
"desc": "by debit card-OTHPG 063444 GOOGLE CLOUD INDIA PVTTHANE-"
},
"source_account": "Assets:Bank"
},
"Expenses:Misc"
],
[
{
"amount": "1 USD",
"date": "2020-01-02",
"key_value_pairs": {
"desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
},
"source_account": "Assets:Bank"
},
"Expenses:Misc"
],
[
{
"amount": "1 USD",
"date": "2020-01-02",
"key_value_pairs": {
"desc": "BULK POSTING- 00000008237 250120 GOOGLE CLOUD INDIA PVT-"
},
"source_account": "Assets:Bank"
},
"Expenses:Misc"
],
[
{
"amount": "500 USD",
"date": "2020-01-02",
"key_value_pairs": {
"desc": "ATM-WD Some Random ATM Machine"
},
"source_account": "Assets:Bank"
},
"Assets:Cash"
],
[
{
"amount": "300 USD",
"date": "2020-01-05",
"key_value_pairs": {
"desc": "Transfer to 1234567890123"
},
"source_account": "Assets:Bank"
},
"Liabilities:JohnDoe"
]
]

0 comments on commit bea4b47

Please sign in to comment.