From cb29f3bd87ffae192a935cdd2dc1c3b75eb9eb99 Mon Sep 17 00:00:00 2001 From: Mario Krajacic Date: Mon, 18 Mar 2024 17:31:58 +0100 Subject: [PATCH] Separate ibkr reader to package --- broker/main.go | 38 +++ fx/main.go | 85 ++----- ibkr/main.go | 373 ++++++++++++++++++++++++++++++ {fx => ibkr}/main_test.go | 12 +- main.go | 474 ++++++-------------------------------- 5 files changed, 508 insertions(+), 474 deletions(-) create mode 100644 broker/main.go rename {fx => ibkr}/main_test.go (72%) diff --git a/broker/main.go b/broker/main.go new file mode 100644 index 0000000..b378d78 --- /dev/null +++ b/broker/main.go @@ -0,0 +1,38 @@ +package broker + +import "time" + +// Tx is a catch-all transaction type +// in this version, it can represent all transaction types except for trades, which need to track quotes at a specific time (Year is not enough) +type Tx struct { + ISIN, Category, Currency string + Amount float64 + Year int +} + +type Trade struct { + // ISIN is the International Securities Identification Number. FIFO method needs to be partitioned by ISIN + ISIN string + // Category is the type of Trade: equity, bond, option, forex, crypto, etc. + Category string + Time time.Time + // Currency is the Currency of the Trade + Currency string `validate:"required,iso4217"` + // Quantity is the number of shares, contracts, or units + Quantity float64 + // Price is the Price per share, contract, or unit + Price float64 +} + +// Statement is an envelope for all relevant broker data from a single file. +// This approach chosen as a middle ground between managing one channel per data type and marshal+switch option +type Statement struct { + Broker string + Filename string + Trades []Trade + FixedIncome, Tax, Fees []Tx +} + +type Reader interface { + Read(filename string) (Statement, error) +} diff --git a/fx/main.go b/fx/main.go index 3b50a93..94f1c7b 100644 --- a/fx/main.go +++ b/fx/main.go @@ -7,7 +7,6 @@ import ( "io" "log" "net/http" - "regexp" "strconv" "strings" "time" @@ -49,70 +48,6 @@ type Rater interface { Rate(currency string, year int) float64 } -func amountFromStringOld(s string) float64 { - if s == "" { - return 0 - - } - // Remove all but numbers, commas and points - re := regexp.MustCompile(`[0-9.,-]`) - ss := strings.Join(re.FindAllString(s, -1), "") - isNeg := ss[0] == '-' - // Find all commas and points - // If none found, return 0, print error - signs := regexp.MustCompile(`[.,]`).FindAllString(ss, -1) - if len(signs) == 0 { - f, err := strconv.ParseFloat(ss, 64) - if err != nil { - fmt.Printf("could not convert %s to number", s) - return 0 - } - - return f - } - - // Use last sign as decimal separator and ignore others - // Find idx and replace whatever sign was to a decimal point - sign := signs[len(signs)-1] - signIdx := strings.LastIndex(ss, sign) - sign = "." - left := regexp.MustCompile(`[0-9]`).FindAllString(ss[:signIdx], -1) - right := ss[signIdx+1:] - n, err := strconv.ParseFloat(strings.Join(append(left, []string{sign, right}...), ""), 64) - if err != nil { - fmt.Printf("could not convert %s to number", s) - return 0 - } - if isNeg { - n = n * -1 - } - return n -} - -// amountFromString formats number strings to float64 type -func amountFromString(s string) float64 { - if s == "" { - return 0 - } - - // Only leave the last decimal point and remove all other points, commas and spaces - lastDec := strings.LastIndex(s, ".") - if lastDec != -1 { - s = strings.Replace(s, ".", "", strings.Count(s, ".")-1) - } - - s = strings.ReplaceAll(s, ",", "") - s = strings.ReplaceAll(s, " ", "") - - // Convert to float - f, err := strconv.ParseFloat(s, 64) - if err != nil { - log.Fatal(err) - } - - return f -} - // url composes the fx exchange rate url for a given currency and year // It accounts for the 2024 currency change and has a default set of currencies to get rates for, to avoid multiple fetches in common currencies func url(currency string, year int) string { @@ -186,7 +121,12 @@ func (fx *Exchange) grabRates(year int, currency string) (err error) { for _, r := range resp.Rates { storeKey := fmt.Sprintf("%s%d", r.Currency, year) - fx.rates[storeKey] = amountFromString(r.Rate) + rate, err := parseFloat(r.Rate) + if err != nil { + log.Println("could not convert rate", r.Rate, "to float") + } + + fx.rates[storeKey] = rate } return @@ -199,6 +139,19 @@ type hnbApiResponse struct { } `json:"rates"` } +func parseFloat(s string) (float64, error) { + // Remove dots, replace commas with dot + s = strings.ReplaceAll(s, ".", "") + s = strings.ReplaceAll(s, ",", ".") + // Convert to float + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, err + } + + return f, nil +} + func New() *Exchange { return &Exchange{grabRetries: 3, rates: make(map[string]float64)} } diff --git a/ibkr/main.go b/ibkr/main.go index f8c6753..97543ff 100644 --- a/ibkr/main.go +++ b/ibkr/main.go @@ -1 +1,374 @@ package ibkr + +import ( + "bufio" + "encoding/csv" + "errors" + "fmt" + "ibkr-report/broker" + "io" + "log" + "os" + "regexp" + "slices" + "strconv" + "strings" + "time" +) + +type instrument struct { + isin string + category string +} + +type reader struct { + header []string + rows []map[string]string + isins map[string]instrument +} + +func (r *reader) read(row []string) { + sections := []string{"Financial Instrument Information", "Trades", "Dividends", "Withholding Tax", "Fees"} + // Ignore if not a section we're interested in + if !slices.Contains(sections, row[0]) { + return + } + + // Update header if new section + if row[1] == "Header" { + r.header = row + return + } + + // If this is financial ISIN information, add symbols to isins map + // otherwise, map the line and store for later + if row[0] == "Financial Instrument Information" { + lm, err := mapIbkrLine(row, r.header) + if err != nil { + return + } + for _, s := range strings.Split(strings.ReplaceAll(lm["Symbol"], " ", ""), ",") { + r.isins[s] = instrument{isin: formatISIN(lm["Security ID"]), category: importCategory(lm["Asset Category"])} + } + return + } + + lm, err := mapIbkrLine(row, r.header) + if err != nil || lm["Header"] != "Data" { + return + } + + r.rows = append(r.rows, lm) + +} + +// Read reads single csv IBKR file +// Filters csv lines by relevant sections and uses *ImportResults to send files to +func Read(filename string) (stmt *broker.Statement, err error) { + file, err := os.Open(filename) + if err != nil { + fmt.Printf("cannot open file %s %v", filename, err) + } + defer func() { + if fErr := file.Close(); fErr != nil { + err = errors.Join(err, fErr) + } + }() + + csvRdr := csv.NewReader(bufio.NewReader(file)) + // Disable record length test in the CSV + csvRdr.FieldsPerRecord = -1 + // Allow quote in unquoted field + csvRdr.LazyQuotes = true + + rdr := reader{isins: make(map[string]instrument)} + for { + row, err := csvRdr.Read() + if err == io.EOF { + break + } + if err != nil { + continue + } + + rdr.read(row) + } + + return rdr.statement(filename) +} + +func (r *reader) statement(filename string) (*broker.Statement, error) { + bs := &broker.Statement{Filename: filename, Broker: "IBKR"} + for _, row := range r.rows { + // All types have a Currency + currency := row["Currency"] + + section := row["Section"] + if section == "Trades" { + if row["Date/Time"] == "" || row["Asset Category"] == "Forex" || row["Symbol"] == "" { + continue + } + + t, err := timeFromExact(row["Date/Time"]) + if err != nil { + continue + } + + bs.Trades = append(bs.Trades, broker.Trade{ + ISIN: r.isins[row["Symbol"]].isin, + Category: r.isins[row["Symbol"]].category, + Time: *t, + Currency: currency, + Quantity: amountFromString(row["Quantity"]), + Price: amountFromString(row["T. Price"]), + }) + + bs.Fees = append(bs.Fees, broker.Tx{ + Category: r.isins[row["Symbol"]].category, + Currency: currency, + Amount: amountFromString(row["Comm/Fee"]), + Year: t.Year(), + }) + + continue + } + + // All other sections only need Year as Time + if row["Date"] == "" { + continue + } + + if section == "Fees" { + bs.Fees = append(bs.Fees, broker.Tx{ + Currency: currency, + Amount: amountFromString(row["Amount"]), + Year: yearFromDate(row["Date"]), + }) + + continue + } + + // Dividends and withholding Tax have the same structure and need to get a symbol from the description + symbol, err := symbolFromDescription(row["Description"]) + if err != nil { + continue + } + + tx := broker.Tx{ + ISIN: r.isins[symbol].isin, + Category: r.isins[symbol].category, + Currency: currency, + Amount: amountFromString(row["Amount"]), + Year: yearFromDate(row["Date"]), + } + + if section == "Dividends" { + bs.FixedIncome = append(bs.FixedIncome, tx) + } else { + bs.Tax = append(bs.Tax, tx) + } + } + + return bs, nil +} + +// symbolFromDescription extracts a symbol from IBKR csv dividend lines +// TODO Check for ISINs +func symbolFromDescription(d string) (string, error) { + if d == "" { + return "", errors.New("empty input") + } + + // This is a dividend or withholding Tax + parensIdx := strings.Index(d, "(") + if parensIdx == -1 { + return "", errors.New("cannot create asset event without symbol") + } + + symbol := strings.ReplaceAll(d[:parensIdx], " ", "") + if symbol == "" { + return "", errors.New("cannot find symbol in description") + } + return symbol, nil +} + +// yearFromDate extracts a Year from IBKR csv date field +func yearFromDate(s string) int { + if s == "" { + return 1900 + } + y, err := strconv.Atoi(s[:4]) + if err != nil { + return 0 + } + return y +} + +// amountFromString formats number strings to float64 type +func amountFromString(s string) float64 { + if s == "" { + return 0 + + } + // Remove all but numbers, commas and points + re := regexp.MustCompile(`[0-9.,-]`) + ss := strings.Join(re.FindAllString(s, -1), "") + isNeg := ss[0] == '-' + // Find all commas and points + // If none found, return 0, print error + signs := regexp.MustCompile(`[.,]`).FindAllString(ss, -1) + if len(signs) == 0 { + f, err := strconv.ParseFloat(ss, 64) + if err != nil { + fmt.Printf("could not convert %s to number", s) + return 0 + } + + return f + } + + // Use last sign as decimal separator and ignore others + // Find idx and replace whatever sign was to a decimal point + sign := signs[len(signs)-1] + signIdx := strings.LastIndex(ss, sign) + sign = "." + left := regexp.MustCompile(`[0-9]`).FindAllString(ss[:signIdx], -1) + right := ss[signIdx+1:] + n, err := strconv.ParseFloat(strings.Join(append(left, []string{sign, right}...), ""), 64) + if err != nil { + fmt.Printf("could not convert %s to number", s) + return 0 + } + if isNeg { + n = n * -1 + } + return n +} + +// timeFromExact extracts time.Time from IBKR csv Time field +func timeFromExact(t string) (*time.Time, error) { + timeStr := strings.Join(strings.Split(t, ","), "") + tm, err := time.Parse("2006-01-02 15:04:05", timeStr) + if err != nil { + return nil, errors.New("could not parse Time") + } + + return &tm, nil +} + +// mapIbkrLine uses a csv line and a related header line to construct a value to field map for easier field lookup while importing lines +func mapIbkrLine(data, header []string) (map[string]string, error) { + if header == nil { + return nil, errors.New("cannot convert to row from empty header") + } + + if data == nil { + return nil, errors.New("no data to map") + } + + if len(header) != len(data) { + return nil, errors.New("header and line length mismatch") + } + + lm := make(map[string]string, len(data)) + for pos, field := range header { + lm[field] = data[pos] + } + + lm["Section"] = data[0] + + return lm, nil +} + +// Adds "US" prefix to US security ISIN codes and removes the 12th check digit +func formatISIN(sID string) string { + if sID == "" || len(sID) < 9 || len(sID) > 12 { + return sID // Not ISIN + } + if len(sID) < 11 { + // US ISIN number. Add country code + sID = "US" + sID + } + if len(sID) == 12 { + // Remove ISIN check digit + return sID[:11] + } + + return sID +} + +func importCategory(c string) string { + if c == "" { + return "" + } + + lc := strings.ToLower(c) + if strings.HasPrefix(lc, "stock") || strings.HasPrefix(lc, "equit") { + return "Equity" + } + + return c +} + +func AmountFromStringOld(s string) float64 { + if s == "" { + return 0 + + } + // Remove all but numbers, commas and points + re := regexp.MustCompile(`[0-9.,-]`) + ss := strings.Join(re.FindAllString(s, -1), "") + isNeg := ss[0] == '-' + // Find all commas and points + // If none found, return 0, print error + signs := regexp.MustCompile(`[.,]`).FindAllString(ss, -1) + if len(signs) == 0 { + f, err := strconv.ParseFloat(ss, 64) + if err != nil { + fmt.Printf("could not convert %s to number", s) + return 0 + } + + return f + } + + // Use last sign as decimal separator and ignore others + // Find idx and replace whatever sign was to a decimal point + sign := signs[len(signs)-1] + signIdx := strings.LastIndex(ss, sign) + sign = "." + left := regexp.MustCompile(`[0-9]`).FindAllString(ss[:signIdx], -1) + right := ss[signIdx+1:] + n, err := strconv.ParseFloat(strings.Join(append(left, []string{sign, right}...), ""), 64) + if err != nil { + fmt.Printf("could not convert %s to number", s) + return 0 + } + if isNeg { + n = n * -1 + } + return n +} + +func AmountFromString(s string) float64 { + if s == "" { + return 0 + } + + // Only leave the last decimal point and remove all other points, commas and spaces + lastDec := strings.LastIndex(s, ".") + if lastDec != -1 { + s = strings.Replace(s, ".", "", strings.Count(s, ".")-1) + } + + s = strings.ReplaceAll(s, ",", "") + s = strings.ReplaceAll(s, " ", "") + + // Convert to float + f, err := strconv.ParseFloat(s, 64) + if err != nil { + log.Fatal(err) + } + + return f +} diff --git a/fx/main_test.go b/ibkr/main_test.go similarity index 72% rename from fx/main_test.go rename to ibkr/main_test.go index 72d357a..d558914 100644 --- a/fx/main_test.go +++ b/ibkr/main_test.go @@ -1,6 +1,8 @@ -package fx +package ibkr -import "testing" +import ( + "testing" +) func Test_amountFromString(t *testing.T) { tests := []struct { @@ -13,7 +15,7 @@ func Test_amountFromString(t *testing.T) { } for _, tt := range tests { - if got := amountFromString(tt.in); got != tt.out { + if got := AmountFromString(tt.in); got != tt.out { t.Errorf("amountFromString(%q) = %v; want %v", tt.in, got, tt.out) } } @@ -21,12 +23,12 @@ func Test_amountFromString(t *testing.T) { func Benchmark_amountFromString(b *testing.B) { for i := 0; i < b.N; i++ { - amountFromString("-79....97,,,,8.97,8 67") + AmountFromString("-79....97,,,,8.97,8 67") } } func Benchmark_amountFromStringOld(b *testing.B) { for i := 0; i < b.N; i++ { - amountFromStringOld("-79....97,,,,8.97,8 67") + AmountFromStringOld("-79....97,,,,8.97,8 67") } } diff --git a/main.go b/main.go index b40dec7..b5bbf50 100644 --- a/main.go +++ b/main.go @@ -2,22 +2,18 @@ package main import ( "bufio" - "encoding/csv" "errors" "fmt" + "ibkr-report/broker" "ibkr-report/fx" - "io" + "ibkr-report/ibkr" "log" "math" - "math/rand" "os" "path/filepath" - "regexp" "runtime" - "slices" "sort" "strconv" - "strings" "sync" "time" ) @@ -28,56 +24,33 @@ func main() { fmt.Println("Finished in", time.Since(t)) }() - files, err := findFiles() - if err != nil { - fmt.Println("Error finding files:", err) - return - } - - r := newReport(newLedger(readFiles(files))) + r := newReport(newLedger(readFiles(findFiles()))) if err := writeFile(r.toRows()); err != nil { log.Fatalf("Error writing report: %v\n", err) } } -// foreign is a representation of capital gains and tax paid from foreign source in a single year +// foreign is a representation of capital gains and Tax paid from foreign source in a single Year type foreign struct { // gains is the total foreign income received gains float64 - // taxPaid is the total tax paid at the foreign source + // taxPaid is the total Tax paid at the foreign source taxPaid float64 } // taxYear represents a single year of taxable income to be reported type taxYear struct { year int - // currency is the currency of the tax year + // currency is the currency of the Tax year // this is to track the Croatian HRK to EUR currency change in 2023 currency string - // realizedPL is the taxable profit from trades, dividends and interest + // realizedPL is the taxable profit from Trades, dividends and interest // matches JOPPD form main input realizedPL float64 - // foreignIncome serves the entries in INO-DOH form, accounting for income tax was fully or partially paid at the foreign source + // foreignIncome serves the entries in INO-DOH form, accounting for income Tax was fully or partially paid at the foreign source foreignIncome map[string]*foreign } -// tx is a catch-all transaction type -// in this version, it can represent all transaction types except for trades, which need to track quotes at a specific time (year is not enough) -type tx struct { - isin, category, currency string - amount float64 - year int -} - -// brokerStatement is an envelope for all relevant broker data from a single file. -// This is what future statement reader need to return. -// This approach chosen as a middle ground between managing one channel per data type and marshal+switch option -type brokerStatement struct { - trades []trade - fixedIncome, tax, fees []tx - ID int -} - type report map[int]*taxYear type pl struct { @@ -92,31 +65,32 @@ type ledger struct { } // findFiles looks for .csv files in the current directory tree, while avoiding duplicates -func findFiles() ([]string, error) { +func findFiles() <-chan string { files := make(map[string]struct{}) - err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { - if filepath.Ext(path) != ".csv" { + ch := make(chan string, 10) + go func() { + defer close(ch) + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if filepath.Ext(path) != ".csv" { + return nil + } + if _, ok := files[path]; !ok { + files[path] = struct{}{} + ch <- path + } return nil + }) + if err != nil { + fmt.Println("Error finding files:", err) } - if _, ok := files[path]; !ok { - files[path] = struct{}{} - } - return nil - }) - if err != nil { - return nil, err - } + }() - var list []string - for k := range files { - list = append(list, k) - } - return list, nil + return ch } -// readFiles creates a brokerStatement for each provided file -func readFiles(files []string) <-chan brokerStatement { - out := make(chan brokerStatement, len(files)) +// readFiles creates a Statement for each provided file +func readFiles(files <-chan string) <-chan *broker.Statement { + out := make(chan *broker.Statement, len(files)) wg := &sync.WaitGroup{} workers := runtime.NumCPU() / 2 @@ -125,16 +99,16 @@ func readFiles(files []string) <-chan brokerStatement { for i := 0; i < workers; i++ { go func() { defer wg.Done() - for _, file := range files { + for file := range files { // read file // transform data into a file report - bs, err := readIbkrStatement(file) + // TODO Factory of broker.Reader to include other brokers + bs, err := ibkr.Read(file) if err != nil { fmt.Println("Error reading file:", err) continue } - bs.ID = rand.Intn(1000) - out <- *bs + out <- bs } }() } @@ -146,28 +120,14 @@ func readFiles(files []string) <-chan brokerStatement { return out } -type trade struct { - // isin is the International Securities Identification Number. FIFO method needs to be partitioned by ISIN - isin string - // category is the type of trade: equity, bond, option, forex, crypto, etc. - category string - time time.Time - // currency is the currency of the trade - currency string `validate:"required,iso4217"` - // quantity is the number of shares, contracts, or units - quantity float64 - // price is the price per share, contract, or unit - price float64 -} - -func fifo(ts []trade, r fx.Rater) []pl { +func fifo(ts []broker.Trade, r fx.Rater) []pl { // FIFO var pls []pl for _, ts := range tradesByISIN(ts) { purchase, sale := 0, 0 for { // find next sale - for sale < len(ts) && ts[sale].quantity >= 0 { + for sale < len(ts) && ts[sale].Quantity >= 0 { sale++ } @@ -175,8 +135,8 @@ func fifo(ts []trade, r fx.Rater) []pl { break } - // Find next purchase. Must have some quantity left to sell - for purchase < sale && ts[purchase].quantity <= 0 { + // Find next purchase. Must have some Quantity left to sell + for purchase < sale && ts[purchase].Quantity <= 0 { purchase++ } @@ -193,70 +153,72 @@ func fifo(ts []trade, r fx.Rater) []pl { return pls } -func tradesByISIN(ts []trade) map[string][]trade { +func tradesByISIN(ts []broker.Trade) map[string][]broker.Trade { sort.Slice(ts, func(i, j int) bool { - return ts[i].time.Before(ts[j].time) + return ts[i].Time.Before(ts[j].Time) }) - grouped := make(map[string][]trade) + grouped := make(map[string][]broker.Trade) for _, t := range ts { - grouped[t.isin] = append(grouped[t.isin], t) + grouped[t.ISIN] = append(grouped[t.ISIN], t) } return grouped } -// plFromTrades, returns the pl from a single purchase and sale trade, as well as a bool indicating if the trade was taxable -func plFromTrades(purchase, sale *trade, r fx.Rater) (pl, bool) { - qtyToSell := math.Min(math.Abs(sale.quantity), math.Abs(purchase.quantity)) - purchase.quantity -= qtyToSell - sale.quantity += qtyToSell +// plFromTrades, returns the pl from a single purchase and sale Trade, as well as a bool indicating if the Trade was taxable +func plFromTrades(purchase, sale *broker.Trade, r fx.Rater) (pl, bool) { + qtyToSell := math.Min(math.Abs(sale.Quantity), math.Abs(purchase.Quantity)) + purchase.Quantity -= qtyToSell + sale.Quantity += qtyToSell - // Convert both currencies with the sale conversion year + // Convert both currencies with the sale conversion Year pl := pl{ - amount: qtyToSell * (sale.price*r.Rate(sale.currency, sale.time.Year()) - purchase.price*r.Rate(purchase.currency, sale.time.Year())), - year: sale.time.Year(), - source: purchase.isin[:2], + amount: qtyToSell * (sale.Price*r.Rate(sale.Currency, sale.Time.Year()) - purchase.Price*r.Rate(purchase.Currency, sale.Time.Year())), + year: sale.Time.Year(), + source: purchase.ISIN[:2], } - return pl, sale.time.Before(purchase.time.AddDate(2, 0, 0)) + return pl, sale.Time.Before(purchase.Time.AddDate(2, 0, 0)) } -func newLedger(statements <-chan brokerStatement) *ledger { - // Store all in ledger to provide to tax report all at once +func newLedger(statements <-chan *broker.Statement) *ledger { + // Store all in ledger to provide to Tax report all at once l := &ledger{deductible: make(map[int]float64)} rtr := fx.New() - var trades []trade + var trades []broker.Trade for stmt := range statements { - l.tax = append(l.tax, plsFromTxs(stmt.tax, rtr)...) - l.profits = append(l.profits, plsFromTxs(stmt.fixedIncome, rtr)...) - trades = append(trades, stmt.trades...) + l.tax = append(l.tax, plsFromTxs(stmt.Tax, rtr)...) + l.profits = append(l.profits, plsFromTxs(stmt.FixedIncome, rtr)...) + trades = append(trades, stmt.Trades...) - for _, fee := range stmt.fees { - if _, ok := l.deductible[fee.year]; !ok { - l.deductible[fee.year] = 0 + for _, fee := range stmt.Fees { + if _, ok := l.deductible[fee.Year]; !ok { + l.deductible[fee.Year] = 0 } - l.deductible[fee.year] += rtr.Rate(fee.currency, fee.year) + l.deductible[fee.Year] += rtr.Rate(fee.Currency, fee.Year) } } - // We have all the trades. Calculate taxable realized profits + // We have all the Trades. Calculate taxable realized profits l.profits = append(l.profits, fifo(trades, rtr)...) return l } -func plsFromTxs(txs []tx, r fx.Rater) []pl { +func plsFromTxs(txs []broker.Tx, r fx.Rater) []pl { pls := make([]pl, 0, len(txs)) for _, tx := range txs { - p := pl{amount: tx.amount * r.Rate(tx.currency, tx.year), year: tx.year} - if tx.isin != "" { - p.source = tx.isin[:2] + rate := r.Rate(tx.Currency, tx.Year) + p := pl{amount: tx.Amount * rate, year: tx.Year} + if tx.ISIN != "" { + p.source = tx.ISIN[:2] } pls = append(pls, p) } + return pls } @@ -268,300 +230,6 @@ func newReport(l *ledger) report { return r } -type instrument struct { - isin string - category string -} - -type reader struct { - header []string - rows []map[string]string - isins map[string]instrument -} - -func (r *reader) read(row []string) { - sections := []string{"Financial Instrument Information", "Trades", "Dividends", "Withholding Tax", "Fees"} - // Ignore if not a section we're interested in - if !slices.Contains(sections, row[0]) { - return - } - - // Update header if new section - if row[1] == "Header" { - r.header = row - return - } - - // If this is financial isin information, add symbols to isins map - // otherwise, map the line and store for later - if row[0] == "Financial Instrument Information" { - lm, err := mapIbkrLine(row, r.header) - if err != nil { - return - } - for _, s := range strings.Split(strings.ReplaceAll(lm["Symbol"], " ", ""), ",") { - r.isins[s] = instrument{isin: formatISIN(lm["Security ID"]), category: importCategory(lm["Asset Category"])} - } - return - } - - lm, err := mapIbkrLine(row, r.header) - if err != nil || lm["Header"] != "Data" { - return - } - - r.rows = append(r.rows, lm) - -} - -// readIbkrStatement reads single csv IBKR file -// Filters csv lines by relevant sections and uses *ImportResults to send files to -func readIbkrStatement(filename string) (stmt *brokerStatement, err error) { - file, err := os.Open(filename) - if err != nil { - fmt.Printf("cannot open file %s %v", filename, err) - } - defer func() { - if fErr := file.Close(); fErr != nil { - err = errors.Join(err, fErr) - } - }() - - csvRdr := csv.NewReader(bufio.NewReader(file)) - // Disable record length test in the CSV - csvRdr.FieldsPerRecord = -1 - // Allow quote in unquoted field - csvRdr.LazyQuotes = true - - rdr := reader{isins: make(map[string]instrument)} - for { - row, err := csvRdr.Read() - if err == io.EOF { - break - } - if err != nil { - continue - } - - rdr.read(row) - } - - return rdr.statement() -} - -func (r *reader) statement() (*brokerStatement, error) { - bs := &brokerStatement{} - for _, row := range r.rows { - // All types have a currency - currency := row["Currency"] - - section := row["Section"] - if section == "Trades" { - if row["Date/Time"] == "" || row["Asset Category"] == "Forex" || row["Symbol"] == "" { - continue - } - - t, err := timeFromExact(row["Date/Time"]) - if err != nil { - continue - } - - bs.trades = append(bs.trades, trade{ - isin: r.isins[row["Symbol"]].isin, - category: r.isins[row["Symbol"]].category, - time: *t, - currency: currency, - quantity: amountFromString(row["Quantity"]), - price: amountFromString(row["T. Price"]), - }) - - bs.fees = append(bs.fees, tx{ - category: r.isins[row["Symbol"]].category, - currency: currency, - amount: amountFromString(row["Comm/Fee"]), - year: t.Year(), - }) - - continue - } - - // All other sections only need year as time - if row["Date"] == "" { - continue - } - - if section == "Fees" { - bs.fees = append(bs.fees, tx{ - currency: currency, - amount: amountFromString(row["Amount"]), - year: yearFromDate(row["Date"]), - }) - - continue - } - - // Dividends and withholding tax have the same structure and need to get a symbol from the description - symbol, err := symbolFromDescription(row["Description"]) - if err != nil { - continue - } - - tx := tx{ - isin: r.isins[symbol].isin, - category: r.isins[symbol].category, - currency: currency, - amount: amountFromString(row["Amount"]), - year: yearFromDate(row["Date"]), - } - - if section == "Dividends" { - bs.fixedIncome = append(bs.fixedIncome, tx) - } else { - bs.tax = append(bs.tax, tx) - } - } - - return bs, nil -} - -// symbolFromDescription extracts a symbol from IBKR csv dividend lines -// TODO Check for ISINs -func symbolFromDescription(d string) (string, error) { - if d == "" { - return "", errors.New("empty input") - } - - // This is a dividend or withholding tax - parensIdx := strings.Index(d, "(") - if parensIdx == -1 { - return "", errors.New("cannot create asset event without symbol") - } - - symbol := strings.ReplaceAll(d[:parensIdx], " ", "") - if symbol == "" { - return "", errors.New("cannot find symbol in description") - } - return symbol, nil -} - -// yearFromDate extracts a year from IBKR csv date field -func yearFromDate(s string) int { - if s == "" { - return 1900 - } - y, err := strconv.Atoi(s[:4]) - if err != nil { - return 0 - } - return y -} - -// amountFromString formats number strings to float64 type -func amountFromString(s string) float64 { - if s == "" { - return 0 - - } - // Remove all but numbers, commas and points - re := regexp.MustCompile(`[0-9.,-]`) - ss := strings.Join(re.FindAllString(s, -1), "") - isNeg := ss[0] == '-' - // Find all commas and points - // If none found, return 0, print error - signs := regexp.MustCompile(`[.,]`).FindAllString(ss, -1) - if len(signs) == 0 { - f, err := strconv.ParseFloat(ss, 64) - if err != nil { - fmt.Printf("could not convert %s to number", s) - return 0 - } - - return f - } - - // Use last sign as decimal separator and ignore others - // Find idx and replace whatever sign was to a decimal point - sign := signs[len(signs)-1] - signIdx := strings.LastIndex(ss, sign) - sign = "." - left := regexp.MustCompile(`[0-9]`).FindAllString(ss[:signIdx], -1) - right := ss[signIdx+1:] - n, err := strconv.ParseFloat(strings.Join(append(left, []string{sign, right}...), ""), 64) - if err != nil { - fmt.Printf("could not convert %s to number", s) - return 0 - } - if isNeg { - n = n * -1 - } - return n -} - -// timeFromExact extracts time.Time from IBKR csv time field -func timeFromExact(t string) (*time.Time, error) { - timeStr := strings.Join(strings.Split(t, ","), "") - tm, err := time.Parse("2006-01-02 15:04:05", timeStr) - if err != nil { - return nil, errors.New("could not parse time") - } - - return &tm, nil -} - -// mapIbkrLine uses a csv line and a related header line to construct a value to field map for easier field lookup while importing lines -func mapIbkrLine(data, header []string) (map[string]string, error) { - if header == nil { - return nil, errors.New("cannot convert to row from empty header") - } - - if data == nil { - return nil, errors.New("no data to map") - } - - if len(header) != len(data) { - return nil, errors.New("header and line length mismatch") - } - - lm := make(map[string]string, len(data)) - for pos, field := range header { - lm[field] = data[pos] - } - - lm["Section"] = data[0] - - return lm, nil -} - -// Adds "US" prefix to US security ISIN codes and removes the 12th check digit -func formatISIN(sID string) string { - if sID == "" || len(sID) < 9 || len(sID) > 12 { - return sID // Not ISIN - } - if len(sID) < 11 { - // US ISIN number. Add country code - sID = "US" + sID - } - if len(sID) == 12 { - // Remove ISIN check digit - return sID[:11] - } - - return sID -} - -func importCategory(c string) string { - if c == "" { - return "" - } - - lc := strings.ToLower(c) - if strings.HasPrefix(lc, "stock") || strings.HasPrefix(lc, "equit") { - return "Equity" - } - - return c -} - func (r report) toRows() [][]string { data := make([][]string, 0, len(r)) for _, year := range r { @@ -576,7 +244,7 @@ func (r report) toRows() [][]string { } } - // sort by year, then report type, then source + // sort by Year, then report type, then source // JOPPD before INO-DOH sort.Slice(data, func(i, j int) bool { if data[i][0] == data[j][0] { @@ -594,7 +262,7 @@ func (r report) toRows() [][]string { func (r report) withWitholdingTax(tax []pl) { for _, pl := range tax { - // Add year to report if not present + // Add Year to report if not present if _, ok := r[pl.year]; !ok { ccy := "EUR" if pl.year < 2023 { @@ -622,7 +290,7 @@ func (r report) withProfits(profits []pl) { r[pl.year] = &taxYear{year: pl.year, foreignIncome: make(map[string]*foreign), currency: ccy} } - // If this is a profit and tax was paid at source, add it to foreign income + // If this is a profit and Tax was paid at source, add it to foreign income fi := r[pl.year].foreignIncome[pl.source] if pl.amount > 0 && fi != nil && fi.taxPaid > 0 { fi.gains += pl.amount @@ -640,7 +308,7 @@ func (r report) withDeductibles(deductible map[int]float64) { } // Balance it out. Do not report income if negative - // Remove year from report if no realized PL and no foreign income + // Remove Year from report if no realized PL and no foreign income for _, year := range r { if year.realizedPL <= 0 { year.realizedPL = 0