From 0b85f82efa5db4fa1db0b98e2552a4a5fef6da7e Mon Sep 17 00:00:00 2001 From: Matthew Bafford Date: Sun, 25 Oct 2020 10:01:01 -0400 Subject: [PATCH] Amazon: Deduct pre-tax adjustments on new invoices Newer style Amazon invoices only have the pre-tax adjustments in the payments table but older invoices have them in the shipments and payments tables (duplicated). This change handles both cases and removes the "Expected, but actual" error when there are pre-tax adjustments on the newer invoices --- beancount_import/source/amazon_invoice.py | 26 +- .../source/amazon/111-1120740-1860260.html | 618 ++++++++++++++++++ .../test_basic/import_results.beancount | 44 ++ .../import_results.beancount | 34 + .../import_results.beancount | 34 + .../test_prediction/import_results.beancount | 44 ++ 6 files changed, 793 insertions(+), 7 deletions(-) create mode 100644 testdata/source/amazon/111-1120740-1860260.html diff --git a/beancount_import/source/amazon_invoice.py b/beancount_import/source/amazon_invoice.py index cace341c..e830ae92 100644 --- a/beancount_import/source/amazon_invoice.py +++ b/beancount_import/source/amazon_invoice.py @@ -386,18 +386,24 @@ def parse_regular_order_invoice(path: str) -> Order: output_fields = dict() output_fields['pretax_adjustments'] = get_adjustments_in_table( payment_table, pretax_adjustment_fields_pattern) - amount = reduce_amounts( - a.amount for a in output_fields['pretax_adjustments']) payment_adjustments = collections.OrderedDict() # type: Dict[str, Amount] + + # older invoices put pre-tax amounts on a per-shipment basis + # new invoices only put pre-tax amounts on the overall payments section + # detect which this is + pretax_amount = reduce_amounts( + a.amount for a in output_fields['pretax_adjustments']) + shipments_pretax_amount = None + if any(s.pretax_adjustments for s in shipments): - expected_amount = reduce_amounts( - a.amount + shipments_pretax_amount = reduce_amounts(a.amount for shipment in shipments - for a in shipment.pretax_adjustments) - if expected_amount != amount: + for a in shipment.pretax_adjustments) + + if shipments_pretax_amount != pretax_amount: errors.append( 'expected total pretax adjustment to be %s, but parsed total is %s' - % (expected_amount, amount)) + % (expected_amount, pretax_amount)) payments_total_adjustments = [] shipments_total_adjustments = [] @@ -455,6 +461,12 @@ def resolve_posttax_adjustments(): expected_total = add_amount(shipments_total_adjustment, reduce_amounts(x.total for x in shipments)) + + # if no shipments pre-tax section, then the expected total isn't accounting + # for the pre-tax adjustments yet since they are only in the grand total section + if shipments_pretax_amount is None: + expected_total = add_amount(expected_total, pretax_amount) + adjusted_grand_total = add_amount(payments_total_adjustment, grand_total) if expected_total != adjusted_grand_total: errors.append('expected grand total is %s, but parsed value is %s' % diff --git a/testdata/source/amazon/111-1120740-1860260.html b/testdata/source/amazon/111-1120740-1860260.html new file mode 100644 index 00000000..6a83eea6 --- /dev/null +++ b/testdata/source/amazon/111-1120740-1860260.html @@ -0,0 +1,618 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Amazon.com - Order 111-1120740-1860260 + + + + + + +
+ +
+
+ Details for Order #111-1120740-1860260 +
+ +

+ + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + Order Placed: + + October 24, 2020 +
+ Amazon.com order number: + 111-1120740-1860260 +
+ Order Total: + $0.00 +
+
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + +
+ + + + +
+
+ Shipping now +
+
+
+ + + + +
+ + + + +
+   +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Items Ordered + + Price +
+ + 1 + + of: + + Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)
+ + + + Sold by: YDM-US (seller profile) + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ + Condition: New
+ + + + + 		+ + $24.99
+ +
+
+
+
+ + + + + +
+ + + +Shipping Address: + + + + +
+ + + + +
+Shipping Speed: + + + + +
+One-Day Shipping +
+ + + + + + + + + + 		+ +
+ +
+
+
+ +
+ + + + + + + + +
+ + + + + + + +
+ + + + +
+
Payment information
 +
+
+ + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Item(s) Subtotal: $24.99
Shipping & Handling:$0.00
Your Coupon Savings:-$3.00
 -----
Total before tax:$21.99
Estimated tax to be collected:$1.32
Gift Card Amount:-$23.31
 -----
Grand Total:$0.00
+ +
+ + + + + + +Payment Method: + + +
+ + + Amazon.com Visa Signature + | Last digits: 1234 +
+ + + + + + Gift Card
+ + + + + + + + + + +
+Billing address + + + +
+
+
+ + + +
+ +
+ +

To view the status of your order, return to Order Summary.

+ +
+ +
+ + +
+ +Conditions of Use +| +Privacy Notice +© 1996-2020, Amazon.com, Inc. or its affiliates +
+ + + + + diff --git a/testdata/source/amazon/test_basic/import_results.beancount b/testdata/source/amazon/test_basic/import_results.beancount index 25aa6a1a..fa3fec89 100644 --- a/testdata/source/amazon/test_basic/import_results.beancount +++ b/testdata/source/amazon/test_basic/import_results.beancount @@ -541,3 +541,47 @@ Expenses:FIXME -27.05 USD amazon_credit_card_description: "Visa ending in 1234" transaction_date: 2020-09-01 + +;; date: 2020-10-24 +;; info: {"filename": "/111-1120740-1860260.html", "type": "text/html"} + +; features: [ +; { +; "amount": "1.32 USD", +; "date": "2020-10-24", +; "key_value_pairs": { +; "amazon_item_description": [ +; "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" +; ] +; }, +; "source_account": "" +; }, +; { +; "amount": "0.00 USD", +; "date": "2020-10-24", +; "key_value_pairs": { +; "amazon_credit_card_description": [ +; "Amazon.com Visa Signature ending in 1234" +; ] +; }, +; "source_account": "" +; } +; ] +2020-10-24 * "Amazon.com" "Order" + amazon_account: "name@domain.com" + amazon_order_id: "111-1120740-1860260" + associated_data0: "{\"description\": \"Amazon order invoice\", \"meta\": [\"amazon_order_id\", \"111-1120740-1860260\"], \"path\": \"/111-1120740-1860260.html\", \"type\": \"text/html\"}" + Expenses:FIXME:A 24.99 USD + amazon_item_condition: "New" + amazon_item_description: "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" + amazon_item_quantity: 1 + amazon_seller: "YDM-US" + Expenses:FIXME:A -3.00 USD + amazon_invoice_description: "Your Coupon Savings" + Expenses:FIXME:A 1.32 USD + amazon_invoice_description: "Sales Tax" + Assets:Gift-Cards:Amazon -23.31 USD + amazon_posttax_adjustment: "Gift Card Amount" + Expenses:FIXME 0.00 USD + amazon_credit_card_description: "Amazon.com Visa Signature ending in 1234" + transaction_date: 2020-10-24 diff --git a/testdata/source/amazon/test_cleared_and_invalid/import_results.beancount b/testdata/source/amazon/test_cleared_and_invalid/import_results.beancount index 11a2a4a2..229069ae 100644 --- a/testdata/source/amazon/test_cleared_and_invalid/import_results.beancount +++ b/testdata/source/amazon/test_cleared_and_invalid/import_results.beancount @@ -222,3 +222,37 @@ Liabilities:Credit-Card -27.05 USD amazon_credit_card_description: "Visa ending in 1234" transaction_date: 2020-09-01 + +;; date: 2020-10-24 +;; info: {"filename": "/111-1120740-1860260.html", "type": "text/html"} + +; features: [ +; { +; "amount": "1.32 USD", +; "date": "2020-10-24", +; "key_value_pairs": { +; "amazon_item_description": [ +; "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" +; ] +; }, +; "source_account": "" +; } +; ] +2020-10-24 * "Amazon.com" "Order" + amazon_account: "name@domain.com" + amazon_order_id: "111-1120740-1860260" + associated_data0: "{\"description\": \"Amazon order invoice\", \"meta\": [\"amazon_order_id\", \"111-1120740-1860260\"], \"path\": \"/111-1120740-1860260.html\", \"type\": \"text/html\"}" + Expenses:FIXME:A 24.99 USD + amazon_item_condition: "New" + amazon_item_description: "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" + amazon_item_quantity: 1 + amazon_seller: "YDM-US" + Expenses:FIXME:A -3.00 USD + amazon_invoice_description: "Your Coupon Savings" + Expenses:FIXME:A 1.32 USD + amazon_invoice_description: "Sales Tax" + Assets:Gift-Cards:Amazon -23.31 USD + amazon_posttax_adjustment: "Gift Card Amount" + Liabilities:Credit-Card 0.00 USD + amazon_credit_card_description: "Amazon.com Visa Signature ending in 1234" + transaction_date: 2020-10-24 diff --git a/testdata/source/amazon/test_credit_card_transactions/import_results.beancount b/testdata/source/amazon/test_credit_card_transactions/import_results.beancount index 8a538dba..46b01ec3 100644 --- a/testdata/source/amazon/test_credit_card_transactions/import_results.beancount +++ b/testdata/source/amazon/test_credit_card_transactions/import_results.beancount @@ -411,3 +411,37 @@ Liabilities:Credit-Card -27.05 USD amazon_credit_card_description: "Visa ending in 1234" transaction_date: 2020-09-01 + +;; date: 2020-10-24 +;; info: {"filename": "/111-1120740-1860260.html", "type": "text/html"} + +; features: [ +; { +; "amount": "1.32 USD", +; "date": "2020-10-24", +; "key_value_pairs": { +; "amazon_item_description": [ +; "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" +; ] +; }, +; "source_account": "" +; } +; ] +2020-10-24 * "Amazon.com" "Order" + amazon_account: "name@domain.com" + amazon_order_id: "111-1120740-1860260" + associated_data0: "{\"description\": \"Amazon order invoice\", \"meta\": [\"amazon_order_id\", \"111-1120740-1860260\"], \"path\": \"/111-1120740-1860260.html\", \"type\": \"text/html\"}" + Expenses:FIXME:A 24.99 USD + amazon_item_condition: "New" + amazon_item_description: "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" + amazon_item_quantity: 1 + amazon_seller: "YDM-US" + Expenses:FIXME:A -3.00 USD + amazon_invoice_description: "Your Coupon Savings" + Expenses:FIXME:A 1.32 USD + amazon_invoice_description: "Sales Tax" + Assets:Gift-Cards:Amazon -23.31 USD + amazon_posttax_adjustment: "Gift Card Amount" + Liabilities:Credit-Card 0.00 USD + amazon_credit_card_description: "Amazon.com Visa Signature ending in 1234" + transaction_date: 2020-10-24 diff --git a/testdata/source/amazon/test_prediction/import_results.beancount b/testdata/source/amazon/test_prediction/import_results.beancount index 9ed2f2c8..ec42f42f 100644 --- a/testdata/source/amazon/test_prediction/import_results.beancount +++ b/testdata/source/amazon/test_prediction/import_results.beancount @@ -252,3 +252,47 @@ Liabilities:Credit-Card -27.05 USD amazon_credit_card_description: "Visa ending in 1234" transaction_date: 2020-09-01 + +;; date: 2020-10-24 +;; info: {"filename": "/111-1120740-1860260.html", "type": "text/html"} + +; features: [ +; { +; "amount": "1.32 USD", +; "date": "2020-10-24", +; "key_value_pairs": { +; "amazon_item_description": [ +; "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" +; ] +; }, +; "source_account": "" +; }, +; { +; "amount": "-23.31 USD", +; "date": "2020-10-24", +; "key_value_pairs": { +; "amazon_posttax_adjustment": [ +; "Gift Card Amount" +; ] +; }, +; "source_account": "" +; } +; ] +2020-10-24 * "Amazon.com" "Order" + amazon_account: "name@domain.com" + amazon_order_id: "111-1120740-1860260" + associated_data0: "{\"description\": \"Amazon order invoice\", \"meta\": [\"amazon_order_id\", \"111-1120740-1860260\"], \"path\": \"/111-1120740-1860260.html\", \"type\": \"text/html\"}" + Expenses:FIXME:A 24.99 USD + amazon_item_condition: "New" + amazon_item_description: "Magnetic USB C Adapter 20Pins Type C Connector, Support USB PD 100W Quick Charge, 10Gb/s Data Transfer and 4K@60 Hz Video Output Compatible with MacBook Pro/Air and More Type C Devices (Grey)" + amazon_item_quantity: 1 + amazon_seller: "YDM-US" + Expenses:FIXME:A -3.00 USD + amazon_invoice_description: "Your Coupon Savings" + Expenses:FIXME:A 1.32 USD + amazon_invoice_description: "Sales Tax" + Expenses:FIXME -23.31 USD + amazon_posttax_adjustment: "Gift Card Amount" + Liabilities:Credit-Card 0.00 USD + amazon_credit_card_description: "Amazon.com Visa Signature ending in 1234" + transaction_date: 2020-10-24