Bugfixes and test fixes for smart_importer support

beancount_import/reconcile.py

@@ -26,6 +26,7 @@
 from .matching import FIXME_ACCOUNT, is_unknown_account, CLEARED_KEY
 
 UNCONFIRMED_ACCOUNT_KEY = 'unconfirmed_account'
+NEW_ACCOUNT_KEY = 'unknown_account'
 display_prediction_explanation = False
 
 classifier_cache_version_number = 1
@@ -729,12 +730,15 @@ def _get_primary_transaction_amount_number(self, transaction: Transaction):
         return None
 
     def _get_unknown_account_names(self, transaction: Transaction):
-        return [posting.account for posting in transaction.postings if posting.meta is not None and NEW_ACCOUNT_KEY in posting.meta]
+        return [posting.account for posting in transaction.postings if posting.meta is not None and UNCONFIRMED_ACCOUNT_KEY in posting.meta]
 
     def _has_unconfirmed_account(self, transaction: Transaction) -> bool:
         return any((posting.meta is not None and posting.meta.get(UNCONFIRMED_ACCOUNT_KEY, False))
                 for posting in transaction.postings)
 
+    def _get_unconfirmed_postings(self, entry: Transaction):
+        return [posting for posting in entry.postings if posting.meta.get(UNCONFIRMED_ACCOUNT_KEY, False)]
+
     def _strip_unconfirmed_account_tags(self, transaction: Transaction):
         '''
             Strips a transaction of meta tags indicating that a posting had a pre-predicted unconfirmed account.
@@ -756,7 +760,6 @@ def _group_predicted_accounts_by_name(self, transaction: Transaction):
         group_numbers = []
         predicted_account_names = []
         existing_groups = {}  # type: Dict[str, int]
-        new_accounts = []
         for posting in transaction.postings:
             if posting.meta is None or not posting.meta.get(UNCONFIRMED_ACCOUNT_KEY, False):
                 continue
@@ -768,12 +771,20 @@ def _group_predicted_accounts_by_name(self, transaction: Transaction):
             group_numbers.append(group_number)
         return predicted_account_names, group_numbers
 
+    """
+    Given a transaction with FIXME account postings, predict the account names for each posting.
+    If any of the postings have an unconfirmed account, then prediction was handled by smart_importer,
+    so remove the posting and return the already predicted account as a prediction.
+    """
     def _get_unknown_account_predictions(self,
-                                         transaction: Transaction) -> List[str]:
+                                         transaction: Transaction) -> Tuple[Transaction, List[str]]:
         if self._has_unconfirmed_account(transaction): 
             # if any of the postings have an unconfirmed account, then prediction was handled by smart_importer
-            predicted_account_names, _ = _group_predicted_accounts_by_name(transaction)
-            return predicted_account_names
+            predicted_account_names, _ = self._group_predicted_accounts_by_name(transaction)
+            # pop the unconfirmed account posting to be added later as predicted account
+            new_postings = [p if not p.meta.get(UNCONFIRMED_ACCOUNT_KEY, False) else p._replace(account=FIXME_ACCOUNT, meta={}) for p in transaction.postings  ]
+            transaction = transaction._replace(postings=new_postings)
+            return transaction, predicted_account_names
         else:
             group_prediction_inputs = self._feature_extractor.extract_unknown_account_group_features(
                 transaction)
@@ -782,7 +793,7 @@ def _get_unknown_account_predictions(self,
                 for prediction_input in group_prediction_inputs
             ]
             group_numbers = training.get_unknown_account_group_numbers(transaction)
-            return [
+            return transaction, [
                 group_predictions[group_number] for group_number in group_numbers
             ]
 
@@ -805,12 +816,8 @@ def _make_candidate_with_substitutions(self,
             unique_id: account
             for unique_id, account in zip(unique_ids, new_accounts)
         }
-        if self._has_unconfirmed_account(transaction):
-            _, group_numbers = self._group_predicted_accounts_by_name(transaction)
-            unknown_names = self._get_unknown_account_names(transaction)
-        else:
-            group_numbers = training.get_unknown_account_group_numbers(transaction)
-            unknown_names = training.get_unknown_account_names(transaction)
+        group_numbers = training.get_unknown_account_group_numbers(transaction)
+        unknown_names = training.get_unknown_account_names(transaction)
         substitutions = [
             AccountSubstitution(
                 unique_name=unique_id,
@@ -883,7 +890,7 @@ def _make_candidates_from_import_result(self, next_pending):
             # Always include the original transaction.
             match_results.append((next_entry, [next_entry]))
             for transaction, used_transactions in match_results:
-                predicted_accounts = self._get_unknown_account_predictions(
+                transaction, predicted_accounts = self._get_unknown_account_predictions(
                     transaction)
                 candidates.append(
                     self._make_candidate_with_substitutions(

beancount_import/source/generic_importer_source.py

@@ -147,7 +147,11 @@ def balance_amounts(txn:Transaction)-> None:
     inventory = SimpleInventory()
     for posting in txn.postings:
         inventory += get_weight(convert_costspec_to_cost(posting))
-    for currency in inventory:
+    unbalanced_currencies = [(currency,v) for currency,v in inventory.items() if v!=0]
+    # posting with no units. Usually added by smart_importer or user
+    empty_posting = any([True if p.units is None else False for p in txn.postings])
+    if unbalanced_currencies and not empty_posting:
+        currency = unbalanced_currencies[0][0]
         txn.postings.append(
             Posting(
                 account=FIXME_ACCOUNT,

testdata/source/generic_importer/test_cost/import_results.beancount

@@ -7,3 +7,4 @@
     date: 2020-01-01
     source_desc: "convert currency"
   Assets:Saving   2 EUR @ 0.5 USD
+    unconfirmed_account: TRUE