Merge branch 'master' into schwabposfix

* master: Add support for options actions: Buy to Open, Buy to Close, Sell to Open, Sell to Close. Add support for Margin Interest. Fixed total of transactions in test CSV for schwab-csv.
jbms · Dec 22, 2020 · 956c76d · 956c76d
2 parents d742fd8 + e9905e9
commit 956c76d
Show file tree

Hide file tree

Showing 5 changed files with 147 additions and 7 deletions.
diff --git a/beancount_import/source/schwab_csv.py b/beancount_import/source/schwab_csv.py
@@ -130,6 +130,11 @@ class SchwabAction(enum.Enum):
     STOCK_PLAN_ACTIVITY = "Stock Plan Activity"
     ADR_MGMT_FEE = "ADR Mgmt Fee"
     FOREIGN_TAX_PAID = "Foreign Tax Paid"
+    MARGIN_INTEREST = "Margin Interest"
+    BUY_TO_OPEN = "Buy to Open"
+    BUY_TO_CLOSE = "Buy to Close"
+    SELL_TO_OPEN = "Sell to Open"
+    SELL_TO_CLOSE = "Sell to Close"
 
 
 @dataclass(frozen=True)
@@ -193,7 +198,7 @@ def get_processed_entry(
             )
         if self.action in (SchwabAction.MONEYLINK_TRANSFER, SchwabAction.JOURNAL):
             return Transfer(**shared_attrs)
-        if self.action == SchwabAction.SELL:
+        if self.action in (SchwabAction.SELL, SchwabAction.SELL_TO_OPEN, SchwabAction.SELL_TO_CLOSE):
             quantity = self.quantity
             assert quantity is not None
             price = self.price
@@ -207,7 +212,7 @@ def get_processed_entry(
                 fees=self.fees,
                 **shared_attrs,
             )
-        if self.action == SchwabAction.BUY:
+        if self.action in (SchwabAction.BUY, SchwabAction.BUY_TO_OPEN, SchwabAction.BUY_TO_CLOSE):
             quantity = self.quantity
             assert quantity is not None
             price = self.price
@@ -224,6 +229,8 @@ def get_processed_entry(
             return Fee(fees_account=fees_account, **shared_attrs)
         if self.action == SchwabAction.FOREIGN_TAX_PAID:
             return TaxPaid(taxes_account=taxes_account, **shared_attrs)
+        if self.action == SchwabAction.MARGIN_INTEREST:
+            return MarginInterest(**shared_attrs)
         assert False, self.action
 
 
@@ -371,6 +378,14 @@ def get_other_account(self) -> str:
     def get_narration_prefix(self) -> str:
         return "INVBANKTRAN"
 
+@dataclass(frozen=True)
+class MarginInterest(TransactionEntry):
+    def get_sub_account(self) -> Optional[str]:
+        return "Cash"
+
+    def get_narration_prefix(self) -> str:
+        return "MARGININTEREST"
+
 
 @dataclass(frozen=True)
 class StockPlanActivity(TransactionEntry):
@@ -502,7 +517,10 @@ def get_postings(self) -> List[Posting]:
         return postings
 
     def get_narration_prefix(self) -> str:
-        return "SELLSTOCK"
+        if self.action in (SchwabAction.SELL_TO_OPEN, SchwabAction.SELL_TO_CLOSE):
+            return "SELLOPT"
+        else:
+            return "SELLSTOCK"
 
     def get_cap_gains_account(self) -> str:
         return f"{self.capital_gains_account}:{self.symbol}"
@@ -565,7 +583,10 @@ def get_postings(self) -> List[Posting]:
         return postings
 
     def get_narration_prefix(self) -> str:
-        return "BUYSTOCK"
+        if self.action in (SchwabAction.BUY_TO_OPEN, SchwabAction.BUY_TO_CLOSE):
+            return "BUYOPT"
+        else:
+            return "BUYSTOCK"
 
     def get_accounts(self) -> List[str]:
         return [self.get_primary_account(), self.get_other_account()]
@@ -683,6 +704,7 @@ def process_positions(
 TAXES_ACCOUNT_KEY = "taxes_account"
 DATE_FORMAT = "%m/%d/%Y"
 TITLE_RE = re.compile(r'"Transactions  for account (?P<account>.+) as of (?P<when>.+)"')
+STRIP_FROM_SYMBOL_RE = re.compile(r'[^\d\w]')
 
 
 class SchwabSourceSpecDict(TypedDict):
@@ -889,7 +911,7 @@ def _load_transactions(filename: str) -> List[RawEntry]:
             assert not found_total_line
             date = _convert_date(row["Date"])
             action = SchwabAction(row["Action"])
-            symbol = row["Symbol"]
+            symbol = STRIP_FROM_SYMBOL_RE.sub("", row["Symbol"])
             description = row["Description"]
             quantity = int(row["Quantity"]) if row["Quantity"] else None
             price = _convert_decimal(row["Price"])
@@ -1024,7 +1046,7 @@ def _load_positions_csv(
         assert not found_account_total, row
         if symbol == "Cash & Cash Investments":
             symbol = "Cash"
-        assert 3 <= len(symbol) <= 4, symbol
+        symbol = STRIP_FROM_SYMBOL_RE.sub("", symbol)
         quantity = _convert_int(row["Quantity"])
         price_d = _convert_decimal(row["Price"])
         price = None if price_d is None else Amount(price_d, currency="USD")

diff --git a/testdata/source/schwab_csv/test_basic/accounts.txt b/testdata/source/schwab_csv/test_basic/accounts.txt
@@ -5,3 +5,6 @@ Assets:Schwab:Intelligent-4321
 Assets:Schwab:Intelligent-4321:Cash
 Assets:Schwab:Intelligent-4321:FNDA
 Assets:Schwab:Intelligent-4321:SCHA
+Assets:Schwab:Intelligent-4321:SMAL040120203200P
+Assets:Schwab:Intelligent-4321:SMAL040320208300C
+Assets:Schwab:Intelligent-4321:SMAL061920201000P
diff --git a/testdata/source/schwab_csv/test_basic/import_results.beancount b/testdata/source/schwab_csv/test_basic/import_results.beancount
@@ -1,3 +1,24 @@
+;; date: 2018-06-28
+;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 16, "type": "text/csv"}
+
+; features: [
+;             {
+;               "amount": "-0.05 USD",
+;               "date": "2018-06-28",
+;               "key_value_pairs": {
+;                 "desc": "INTEREST 05/30THRU 06/27",
+;                 "schwab_action": "Margin Interest"
+;               },
+;               "source_account": "Assets:Schwab:Intelligent-4321:Cash"
+;             }
+;           ]
+2018-06-28 * "MARGININTEREST - INTEREST 05/30THRU 06/27"
+  Assets:Schwab:Intelligent-4321:Cash  -0.05 USD
+    date: 2018-06-28
+    schwab_action: "Margin Interest"
+    source_desc: "INTEREST 05/30THRU 06/27"
+  Expenses:FIXME                        0.05 USD
+
 ;; date: 2019-06-11
 ;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 15, "type": "text/csv"}
 
@@ -63,6 +84,83 @@
     source_desc: "VANECK VECTORS J.P. MORGAN EM LOCAL CURRENCYBOND ETF"
   Income:Dividend:Schwab:EMLC          -51.77 USD
 
+;; date: 2020-03-17
+;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 17, "type": "text/csv"}
+
+; features: []
+2020-03-17 * "BUYOPT - PUT SMALL CAP INDEX $32 EXP 04/01/20"
+  Assets:Schwab:Intelligent-4321:SMAL040120203200P         1 SMAL040120203200P {12.53 USD}
+    date: 2020-03-17
+    schwab_action: "Buy to Open"
+    source_desc: "PUT SMALL CAP INDEX $32 EXP 04/01/20"
+  Assets:Schwab:Intelligent-4321:Cash               -1253.65 USD
+    date: 2020-03-17
+    schwab_action: "Buy to Open"
+    source_desc: "PUT SMALL CAP INDEX $32 EXP 04/01/20"
+  Expenses:Brokerage-Fees:Schwab                        0.65 USD
+
+;; date: 2020-03-18
+;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 18, "type": "text/csv"}
+
+; features: []
+2020-03-18 * "SELLOPT - PUT SMALL CAP INDEX $32 EXP 04/01/20"
+  Assets:Schwab:Intelligent-4321:SMAL040120203200P       -1 SMAL040120203200P {} @ 15.66 USD
+    date: 2020-03-18
+    schwab_action: "Sell to Close"
+    source_desc: "PUT SMALL CAP INDEX $32 EXP 04/01/20"
+  Income:Capital-Gains:Schwab:SMAL040120203200P
+  Assets:Schwab:Intelligent-4321:Cash               1565.32 USD
+    date: 2020-03-18
+    schwab_action: "Sell to Close"
+    source_desc: "PUT SMALL CAP INDEX $32 EXP 04/01/20"
+  Expenses:Brokerage-Fees:Schwab                       0.68 USD
+
+;; date: 2020-03-27
+;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 19, "type": "text/csv"}
+
+; features: []
+2020-03-27 * "SELLOPT - PUT SMALL CAP INDEX $10 EXP 06/19/20"
+  Assets:Schwab:Intelligent-4321:SMAL061920201000P      -1 SMAL061920201000P {} @ 8.48 USD
+    date: 2020-03-27
+    schwab_action: "Sell to Open"
+    source_desc: "PUT SMALL CAP INDEX $10 EXP 06/19/20"
+  Income:Capital-Gains:Schwab:SMAL061920201000P
+  Assets:Schwab:Intelligent-4321:Cash               847.33 USD
+    date: 2020-03-27
+    schwab_action: "Sell to Open"
+    source_desc: "PUT SMALL CAP INDEX $10 EXP 06/19/20"
+  Expenses:Brokerage-Fees:Schwab                      0.67 USD
+
+;; date: 2020-03-30
+;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 21, "type": "text/csv"}
+
+; features: []
+2020-03-30 * "BUYOPT - CALL SMALL CAP INDEX $83 EXP 04/03/20"
+  Assets:Schwab:Intelligent-4321:SMAL040320208300C       2 SMAL040320208300C {0.49 USD}
+    date: 2020-03-30
+    schwab_action: "Buy to Open"
+    source_desc: "CALL SMALL CAP INDEX $83 EXP 04/03/20"
+  Assets:Schwab:Intelligent-4321:Cash               -99.34 USD
+    date: 2020-03-30
+    schwab_action: "Buy to Open"
+    source_desc: "CALL SMALL CAP INDEX $83 EXP 04/03/20"
+  Expenses:Brokerage-Fees:Schwab                      1.34 USD
+
+;; date: 2020-06-12
+;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 20, "type": "text/csv"}
+
+; features: []
+2020-06-12 * "BUYOPT - PUT SMALL CAP INDEX $10 EXP 06/19/20"
+  Assets:Schwab:Intelligent-4321:SMAL061920201000P      1 SMAL061920201000P {0.06 USD}
+    date: 2020-06-12
+    schwab_action: "Buy to Close"
+    source_desc: "PUT SMALL CAP INDEX $10 EXP 06/19/20"
+  Assets:Schwab:Intelligent-4321:Cash               -6.65 USD
+    date: 2020-06-12
+    schwab_action: "Buy to Close"
+    source_desc: "PUT SMALL CAP INDEX $10 EXP 06/19/20"
+  Expenses:Brokerage-Fees:Schwab                     0.65 USD
+
 ;; date: 2020-08-24
 ;; info: {"filename": "<testdata>/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV", "line": 10, "type": "text/csv"}
 
@@ -283,6 +381,11 @@
 ;; date: 2020-11-15
 ;; info: {"filename": "<testdata>/test_basic/positions/All-Accounts-Positions-2020-11-15.CSV", "line": 13, "type": "text/csv"}
 
+2020-11-15 balance Assets:Schwab:Intelligent-4321:SMAL040120203200P 1 SMAL040120203200P
+
+;; date: 2020-11-15
+;; info: {"filename": "<testdata>/test_basic/positions/All-Accounts-Positions-2020-11-15.CSV", "line": 14, "type": "text/csv"}
+
 2020-11-15 balance Assets:Schwab:Intelligent-4321:Cash             2000.00 USD
 
 ;; date: 2020-11-15
@@ -295,6 +398,11 @@
 
 2020-11-15 price SCHA                                78.97 USD
 
+;; date: 2020-11-15
+;; info: {"filename": "<testdata>/test_basic/positions/All-Accounts-Positions-2020-11-15.CSV", "line": 13, "type": "text/csv"}
+
+2020-11-15 price SMAL040120203200P                   12.53 USD
+
 ;; date: 2020-12-21
 ;; info: {"filename": "<testdata>/test_basic/positions/Intelligent-Positions-2020-11-30.CSV", "line": 4, "type": "text/csv"}
 

diff --git a/testdata/source/schwab_csv/test_basic/positions/All-Accounts-Positions-2020-11-15.CSV b/testdata/source/schwab_csv/test_basic/positions/All-Accounts-Positions-2020-11-15.CSV
@@ -10,5 +10,6 @@
 "Symbol","Description","Quantity","Price","Price Change $","Price Change %","Market Value","Day Change $","Day Change %","Cost Basis","Gain/Loss $","Gain/Loss %","Reinvest Dividends?","Capital Gains?","% Of Account","Dividend Yield","Last Dividend","Ex-Dividend Date","P/E Ratio","52 Week Low","52 Week High","Volume","Intrinsic Value","In The Money","Security Type",
 "FNDA","SCHWAB FUNDAMENTAL US SMALL COM ETF","100","$38.30","$1.00","+2.68%","$3830.00","$100.00","+2.68%","$3000.00","$830.00","+27.66%","No","--","3.15%","+1.4%","$0.14","9/23/2020","--","$21.80","$40.88","167,797","--","--","ETFs & Closed End Funds",
 "SCHA","SCHWAB US SMALL CAP ETF","100","$78.97","$1.78","+2.31%","$7,897.00","$178.00","+2.31%","$7,000.00","$897.00","+12.81%","No","--","5.21%","+1.3%","$0.28","9/23/2020","--","$43.05","$79.71","1,709,156","--","--","ETFs & Closed End Funds",
+"SMAL 04/01/2020 32.00 P","PUT SMALL CAP INDEX $32 EXP 04/01/20",1,"$12.53","$0","0%","$1253","$0.00","0%","$1253","$0","0%","No","--","1%","0%","$0","--","--","$1","$100",1000,"--","TRUE","ETFs & Closed End Funds"
 "Cash & Cash Investments","--","--","--","--","--","$2,000.00","$50.00","+2.56%","--","--","--","--","--","13.6%","--","--","--","--","--","--","--","--","--","Cash and Money Market",
 "Account Total","--","--","--","--","--","$13,727.00","$328.00","+2.38%","$12,000.00","$1,727.00","+12.58%","--","--","--","--","--","--","--","--","--","--",
diff --git a/...ta/source/schwab_csv/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV b/...ta/source/schwab_csv/test_basic/transactions/Intelligent_Transactions_20201115-180120.CSV
@@ -14,4 +14,10 @@
 "06/11/2019","Qualified Dividend","NOK","NOKIA CORP FSPONSORED ADR 1 ADR REPS 1 ORD SHS","","","","$6.32",
 "06/11/2019","ADR Mgmt Fee","NOK","NOKIA CORP FSPONSORED ADR 1 ADR REPS 1 ORD SHS","","","","-$0.47",
 "06/11/2019","Foreign Tax Paid","NOK","NOKIA CORP FSPONSORED ADR 1 ADR REPS 1 ORD SHS","","","","-$1.05"
-Transactions Total,"","","","","","",-$8366.94
+"06/28/2018","Margin Interest","","INTEREST 05/30THRU 06/27","","","","-$0.05"
+"03/17/2020","Buy to Open","SMAL 04/01/2020 32.00 P","PUT SMALL CAP INDEX $32 EXP 04/01/20","1","$12.53","$0.65","-$1253.65"
+"03/18/2020","Sell to Close","SMAL 04/01/2020 32.00 P","PUT SMALL CAP INDEX $32 EXP 04/01/20","1","$15.66","$0.68","$1565.32"
+"03/27/2020","Sell to Open","SMAL 06/19/2020 10.00 P","PUT SMALL CAP INDEX $10 EXP 06/19/20",1,"$8.48","$0.67","$847.33"
+"06/12/2020","Buy to Close","SMAL 06/19/2020 10.00 P","PUT SMALL CAP INDEX $10 EXP 06/19/20",1,"$0.06","$0.65","-$6.65"
+"03/30/2020","Buy to Open","SMAL 04/03/2020 83.00 C","CALL SMALL CAP INDEX $83 EXP 04/03/20",2,"$0.49","$1.34","-$99.34"
+Transactions Total,"","","","","","",-$9620.64