Add UNCONFIRMED to externally predicted postings of imported entries

beancount_import/reconcile.py

@@ -25,6 +25,8 @@
 
 from .matching import FIXME_ACCOUNT, is_unknown_account, CLEARED_KEY
 
+UNCONFIRMED_ACCOUNT_KEY = 'unconfirmed_account'
+NEW_ACCOUNT_KEY = 'unknown_account'
 display_prediction_explanation = False
 
 classifier_cache_version_number = 1
@@ -584,7 +586,8 @@ def _get_fixme_transactions(self):
             if isinstance(entry, Transaction):
                 if any(
                         is_unknown_account(posting.account)
-                        for posting in entry.postings):
+                        for posting in entry.postings) \
+                   or self._has_unconfirmed_account(entry):
                     output.append(entry)
         return output
 
@@ -726,18 +729,73 @@ def _get_primary_transaction_amount_number(self, transaction: Transaction):
                 return -source_posting.units.number
         return None
 
+    def _get_unknown_account_names(self, transaction: Transaction):
+        return [posting.account for posting in transaction.postings if posting.meta is not None and UNCONFIRMED_ACCOUNT_KEY in posting.meta]
+
+    def _has_unconfirmed_account(self, transaction: Transaction) -> bool:
+        return any((posting.meta is not None and posting.meta.get(UNCONFIRMED_ACCOUNT_KEY, False))
+                for posting in transaction.postings)
+
+    def _get_unconfirmed_postings(self, entry: Transaction):
+        return [posting for posting in entry.postings if posting.meta.get(UNCONFIRMED_ACCOUNT_KEY, False)]
+
+    def _strip_unconfirmed_account_tags(self, transaction: Transaction):
+        '''
+            Strips a transaction of meta tags indicating that a posting had a pre-predicted unconfirmed account.
+            Leaves postings with FIXME account unchanged.
+        '''
+        for posting in transaction.postings:
+            if posting.account == FIXME_ACCOUNT:
+                continue
+            if posting.meta is not None and UNCONFIRMED_ACCOUNT_KEY in posting.meta:
+                posting.meta.pop(UNCONFIRMED_ACCOUNT_KEY)
+
+    def _group_predicted_accounts_by_name(self, transaction: Transaction):
+        '''
+            Takes a list of postings with candidate account names,
+            and groups them into groups that should share the same exact account.
+            Expects each predicted posting to have an UNCONFIRMED_ACCOUNT_KEY meta field.
+        '''
+        num_groups = 0
+        group_numbers = []
+        predicted_account_names = []
+        existing_groups = {}  # type: Dict[str, int]
+        for posting in transaction.postings:
+            if posting.meta is None or not posting.meta.get(UNCONFIRMED_ACCOUNT_KEY, False):
+                continue
+            group_number = existing_groups.setdefault(posting.account,
+                                                        num_groups)
+            predicted_account_names.append(posting.account)
+            if group_number == num_groups:
+                num_groups += 1
+            group_numbers.append(group_number)
+        return predicted_account_names, group_numbers
+
+    """
+    Given a transaction with FIXME account postings, predict the account names for each posting.
+    If any of the postings have an unconfirmed account, then prediction was handled by smart_importer,
+    so remove the posting and return the already predicted account as a prediction.
+    """
     def _get_unknown_account_predictions(self,
-                                         transaction: Transaction) -> List[str]:
-        group_prediction_inputs = self._feature_extractor.extract_unknown_account_group_features(
-            transaction)
-        group_predictions = [
-            self.predict_account(prediction_input)
-            for prediction_input in group_prediction_inputs
-        ]
-        group_numbers = training.get_unknown_account_group_numbers(transaction)
-        return [
-            group_predictions[group_number] for group_number in group_numbers
-        ]
+                                         transaction: Transaction) -> Tuple[Transaction, List[str]]:
+        if self._has_unconfirmed_account(transaction): 
+            # if any of the postings have an unconfirmed account, then prediction was handled by smart_importer
+            predicted_account_names, _ = self._group_predicted_accounts_by_name(transaction)
+            # pop the unconfirmed account posting to be added later as predicted account
+            new_postings = [p if not p.meta.get(UNCONFIRMED_ACCOUNT_KEY, False) else p._replace(account=FIXME_ACCOUNT, meta={}) for p in transaction.postings  ]
+            transaction = transaction._replace(postings=new_postings)
+            return transaction, predicted_account_names
+        else:
+            group_prediction_inputs = self._feature_extractor.extract_unknown_account_group_features(
+                transaction)
+            group_predictions = [
+                self.predict_account(prediction_input)
+                for prediction_input in group_prediction_inputs
+            ]
+            group_numbers = training.get_unknown_account_group_numbers(transaction)
+            return transaction, [
+                group_predictions[group_number] for group_number in group_numbers
+            ]
 
     def _make_candidate_with_substitutions(self,
                                            transaction: Transaction,
@@ -832,7 +890,7 @@ def _make_candidates_from_import_result(self, next_pending):
             # Always include the original transaction.
             match_results.append((next_entry, [next_entry]))
             for transaction, used_transactions in match_results:
-                predicted_accounts = self._get_unknown_account_predictions(
+                transaction, predicted_accounts = self._get_unknown_account_predictions(
                     transaction)
                 candidates.append(
                     self._make_candidate_with_substitutions(
@@ -920,6 +978,7 @@ def accept_candidate(self, candidate: Candidate, ignore=False) -> AcceptCandidat
         for entry in new_entries:
             if isinstance(entry, Transaction):
                 self.posting_db.add_transaction(entry)
+                self._strip_unconfirmed_account_tags(entry)
 
         self._extract_training_examples(new_entries)
 

beancount_import/source/generic_importer_source.py

@@ -31,6 +31,7 @@
 from .description_based_source import DescriptionBasedSource, get_pending_and_invalid_entries
 from .mint import _get_key_from_posting
 
+UNCONFIRMED = "unconfirmed_account"
 
 class ImporterSource(DescriptionBasedSource):
     def __init__(self,
@@ -66,7 +67,7 @@ def prepare(self, journal: 'JournalEditor', results: SourceResults) -> None:
             hashed_entries = OrderedDict() #type: Dict[Hashable, Directive]
             for entry in f_entries:
                 key_ = self._get_key_from_imported_entry(entry)
-                self._add_description(entry)
+                self._add_description_and_unconfirmed_account_key(entry)
                 hashed_entries.setdefault(key_, []).append(entry)
             # deduplicate across statements
             for key_ in hashed_entries:
@@ -86,24 +87,20 @@ def prepare(self, journal: 'JournalEditor', results: SourceResults) -> None:
             make_import_result=self._make_import_result,
             results=results)
 
-    def _add_description(self, entry: Transaction):
+    def _add_description_and_unconfirmed_account_key(self, entry: Transaction):
         if not isinstance(entry, Transaction): return None
-        postings = entry.postings #type: List[Posting]
-        to_mutate = []
-        for i, posting in enumerate(postings):
-            if posting.account != self.account: continue
-            if isinstance(posting.meta, dict):
+        for i, posting in enumerate(entry.postings):
+            if not isinstance(posting.meta, dict):
+                # replace posting with a clone with meta dict
+                entry.postings[i] = posting._replace(meta={})
+                posting = entry.postings[i]
+            if posting.account == self.account:
+                # add description
                 posting.meta["source_desc"] = entry.narration
                 posting.meta["date"] = entry.date
-                break
             else:
-                to_mutate.append(i)
-                break
-        for i in to_mutate:
-            p = postings.pop(i)
-            p = Posting(p.account, p.units, p.cost, p.price, p.flag,
-                        {"source_desc":entry.narration, "date": entry.date})
-            postings.insert(i, p)
+                # add unconfirmed account key
+                posting.meta[UNCONFIRMED] = True
 
     def _get_source_posting(self, entry:Transaction) -> Optional[Posting]:
         for posting in entry.postings:
@@ -150,7 +147,11 @@ def balance_amounts(txn:Transaction)-> None:
     inventory = SimpleInventory()
     for posting in txn.postings:
         inventory += get_weight(convert_costspec_to_cost(posting))
-    for currency in inventory:
+    unbalanced_currencies = [(currency,v) for currency,v in inventory.items() if round(v, 5)!=0]
+    # posting with no units. Usually added by smart_importer or user
+    empty_posting = any([True if p.units is None else False for p in txn.postings])
+    if unbalanced_currencies and not empty_posting:
+        currency = unbalanced_currencies[0][0]
         txn.postings.append(
             Posting(
                 account=FIXME_ACCOUNT,

testdata/source/generic_importer/test_cost/import_results.beancount

@@ -7,3 +7,4 @@
     date: 2020-01-01
     source_desc: "convert currency"
   Assets:Saving   2 EUR @ 0.5 USD
+    unconfirmed_account: TRUE