Merge into main, resolve conflicts

.gitignore

@@ -20,3 +20,5 @@ playground.txt
 tmp/
 dist/
 .dccache
+
+dist/

fx.go

@@ -13,46 +13,55 @@ import (
 	"time"
 )
 
-type Rates map[int]map[string]float64
+type Rates struct {
+	l     *sync.Mutex
+	rates map[int]map[string]float64
+}
 
-func (r Rates) Rate(ccy string, yr int) float64 {
-	return r[yr][ccy] // TODO Check errors
+func (r *Rates) Rate(ccy string, yr int) float64 {
+	return r.rates[yr][ccy] // Should not be any errors here. Throw if not fetched correctly
 }
 
-func NewFxRates(currencies []string, years []int) (Rates, error) {
-	t := time.Now()
-	r := make(Rates, len(years))
+func (r *Rates) setRates(y int, rates map[string]float64) {
+	r.l.Lock()
+	defer r.l.Unlock()
+	r.rates[y] = rates
+}
 
-	var m sync.Mutex
+// NewFxRates creates a new Rates struct by fetching currency exhange rates for provided years and currencies
+// TODO Do not fetch in New
+func NewFxRates(currencies []string, years []int) (*Rates, error) {
+	r := &Rates{
+		l:     new(sync.Mutex),
+		rates: map[int]map[string]float64{},
+	}
 	var wg sync.WaitGroup
-	wg.Add(len(years))
-	for _, y := range years {
-		go func(r Rates, y int, m *sync.Mutex, wg *sync.WaitGroup) {
+	workers := maxWorkers
+	if len(years) < maxWorkers {
+		workers = len(years)
+	}
+	wg.Add(workers)
+	yrs := make(chan int)
+	for w := 0; w < workers; w++ {
+		go func(r *Rates, yrs <-chan int, wg *sync.WaitGroup) {
 			defer wg.Done()
-			rates, err := grabHRKRates(y, currencies, 3)
-			if err != nil {
-				fmt.Println(err)
-				fmt.Printf("cannot get rates for year %d", y)
-				return
+			for y := range yrs {
+				rates, err := grabHRKRates(y, currencies, 3)
+				if err != nil {
+					log.Fatalln("failed getting currency exchange rates from hnb.hr. Please try again later")
+				}
+				r.setRates(y, rates)
 			}
-			m.Lock()
-			defer m.Unlock()
-			r[y] = rates
-		}(r, y, &m, &wg)
+		}(r, yrs, &wg)
 	}
-	wg.Wait()
-
-	fmt.Println("Rates fetched in:", time.Since(t))
-	return r, nil
-}
 
-func (r Rates) Print() {
-	fmt.Println("Fx")
-	for y := range r {
-		for c := range r[y] {
-			fmt.Printf("%d %s %g\n", y, c, r[y][c])
-		}
+	for _, y := range years {
+		yrs <- y
 	}
+
+	close(yrs)
+	wg.Wait()
+	return r, nil
 }
 
 // TODO Other currencies https://ec.europa.eu/info/funding-tenders/procedures-guidelines-tenders/information-contractors-and-beneficiaries/exchange-rate-inforeuro_en
@@ -65,16 +74,18 @@ type ratesResponse struct {
 	Rates []rateResponse `json:"rates"`
 }
 
+// grabHRKRates fetches HRK exchange rates for a list of currencies in a provided year from hnb.hr
 func grabHRKRates(year int, c []string, retries int) (map[string]float64, error) {
 	if year <= 1900 {
 		log.Fatal("Cannot get currency rates for a year before 1901")
 	}
 
 	// TODO Other currencies than HRK
-	date := LastDateForYear(year)
+	date := lastDateForYear(year)
 
 	// url := fmt.Sprintf("https://api.hnb.hr/tecajn/v1?datum-od=%s&datum-do=%s", from.Format("2006-01-02"), to.Format("2006-01-02"))
-	url := fmt.Sprintf("https://api.hnb.hr/tecajn/v1?datum=%s", date.Format("2006-01-02"))
+	baseUrl := "https://api.hnb.hr/tecajn/v1"
+	url := fmt.Sprintf(baseUrl+"?datum=%s", date.Format("2006-01-02"))
 	for _, curr := range c {
 		url = url + "&valuta=" + curr
 	}
@@ -102,7 +113,6 @@ func grabHRKRates(year int, c []string, retries int) (map[string]float64, error)
 	err3 := json.Unmarshal(contents, &resp.Rates)
 	if err3 != nil {
 		fmt.Println("whoops:", err3)
-		// outputs: whoops: <nil>
 	}
 
 	rm := make(map[string]float64, len(c))
@@ -113,7 +123,10 @@ func grabHRKRates(year int, c []string, retries int) (map[string]float64, error)
 	return rm, nil
 }
 
-func LastDateForYear(y int) (d time.Time) {
+// lastDateForYear calculates last day of the year for the input
+// If year is current year, returns today
+// Return time set to UTC
+func lastDateForYear(y int) (d time.Time) {
 	if y == time.Now().Year() {
 		d = time.Now().UTC()
 	} else {
@@ -123,11 +136,11 @@ func LastDateForYear(y int) (d time.Time) {
 	return
 }
 
+// formatApiRate formats the api response currency rate received from hnb.hr
 func formatApiRate(r string) float64 {
 	s := strings.ReplaceAll(strings.ReplaceAll(r, ".", ""), ",", ".")
-
 	if s == "" {
-		log.Fatalf("Cannot create amount from %s", s)
+		log.Fatalf("cannot create amount from %s", s)
 	}
 	s = strings.ReplaceAll(s, ",", "")
 	v, err := strconv.ParseFloat(s, 64)

go.mod

@@ -2,12 +2,13 @@ module ibkr-report
 
 go 1.18
 
+require github.com/xuri/excelize/v2 v2.6.0
+
 require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/richardlehane/mscfb v1.0.4 // indirect
 	github.com/richardlehane/msoleps v1.0.1 // indirect
 	github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8 // indirect
-	github.com/xuri/excelize/v2 v2.6.0 // indirect
 	github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect
 	golang.org/x/crypto v0.0.0-20220408190544-5352b0902921 // indirect
 	golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect

go.sum

@@ -1,12 +1,15 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
 github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
 github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o=
 github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8 h1:3X7aE0iLKJ5j+tz58BpvIZkXNV7Yq4jC93Z/rbN2Fxk=
 github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
@@ -16,6 +19,7 @@ github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 h1:OAmKAfT06//esDdpi/DZ8Q
 github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
 golang.org/x/crypto v0.0.0-20220408190544-5352b0902921 h1:iU7T1X1J6yxDr0rda54sWGkHgOp5XJrqm79gcNlC2VM=
 golang.org/x/crypto v0.0.0-20220408190544-5352b0902921/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 h1:EN5+DfgmRMvRUrMGERW2gQl3Vc+Z7ZMnI/xdEpPSf0c=
@@ -31,4 +35,5 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

import_results.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"sync"
 	"time"
 )
 
@@ -37,6 +38,8 @@ type ImportResults struct {
 	// Mapped unique currencies and years found in any imported events
 	currencies map[string]bool
 	years      map[int]bool
+
+	l *sync.Mutex
 }
 
 // Domicile extracts the instrument country code used fo the ForeignIncome tax report
@@ -73,21 +76,29 @@ func NewImportResults() *ImportResults {
 		years: map[int]bool{
 			time.Now().Year(): true,
 		},
+		l: new(sync.Mutex),
 	}
 }
 
 func (r *ImportResults) AddInstrumentInfo(symbols []string, cat string) {
+	r.l.Lock()
+	defer r.l.Unlock()
 	a := r.assets.bySymbols(symbols...)
 	if a.Category != "" || cat == "" {
 		return
 	}
 	a.Category = cat
 }
-func (r *ImportResults) AddTrade(sym, ccy string, tm time.Time, qty, price, fee float64) {
+func (r *ImportResults) AddTrade(sym, ccy string, tm *time.Time, qty, price, fee float64) {
+	if sym == "" || ccy == "" || tm == nil || qty*price == 0 {
+		return
+	}
+	r.l.Lock()
+	defer r.l.Unlock()
 	a := r.assets.bySymbols(sym)
 	t := Trade{}
 	t.Currency = ccy
-	t.Time = tm
+	t.Time = *tm
 	t.Quantity = qty
 	t.Price = price
 	t.Fee = fee
@@ -98,6 +109,11 @@ func (r *ImportResults) AddTrade(sym, ccy string, tm time.Time, qty, price, fee
 
 }
 func (r *ImportResults) AddDividend(sym, ccy string, yr int, amt float64, isTax bool) {
+	if sym == "" || ccy == "" || yr == 0 || amt == 0 {
+		return
+	}
+	r.l.Lock()
+	defer r.l.Unlock()
 	a := r.assets.bySymbols(sym)
 	d := Transaction{}
 	d.Currency = ccy
@@ -113,6 +129,11 @@ func (r *ImportResults) AddDividend(sym, ccy string, yr int, amt float64, isTax
 	r.years[yr] = true
 }
 func (r *ImportResults) AddFee(ccy string, amt float64, yr int) {
+	if yr == 0 || amt == 0 || ccy == "" {
+		return
+	}
+	r.l.Lock()
+	defer r.l.Unlock()
 	f := Transaction{}
 	f.Currency = ccy
 	f.Year = yr
@@ -121,7 +142,6 @@ func (r *ImportResults) AddFee(ccy string, amt float64, yr int) {
 
 	r.currencies[ccy] = true
 	r.years[yr] = true
-
 }
 
 // bySymbols func looks up any assets already imported by incoming symbols
@@ -130,55 +150,80 @@ func (r *ImportResults) AddFee(ccy string, amt float64, yr int) {
 //
 // If incoming symbols are matched with more than one distinct asset, merges conflict and uses the first found match
 func (as assets) bySymbols(ss ...string) *AssetImport {
-	if len(ss) == 0 {
+	if len(ss) < 1 {
 		return nil
 	}
 
-	var base *AssetImport
-	newSymbols := make([]string, 0, len(ss))
+	if len(ss) == 1 {
+		match, ok := as[ss[0]]
+		if !ok {
+			a := &AssetImport{Instrument: Instrument{Symbols: ss}}
+			as[ss[0]] = a
+			return a
+		}
+
+		return match
+	}
 
+	// Target is the merged asset combining all of the info of incoming symbols and previously included symbols for an instrument
+	// Processed symbols used to avoid processing symbols twice
+	// Unmatched symbols slcie stores incoming symbols not matched with any existing assets. Once any match has been found, not needed anymore
+	var target *AssetImport
+	processed := make(map[string]bool, len(ss))
+	unmatched := make([]string, 0, len(ss))
 	for _, s := range ss {
-		if s == "" {
+		// Skip processinf the same symbol twice
+		if _, ok := processed[s]; ok {
 			continue
 		}
 
+		// Find existing match
+		// If first match found (target), set target and add include all unmatched symbols
+		// If match not found, but target already found, just add the symbol
+		// Skip if match found and equal to the target
+		// Merge info to target if match found not equal
 		match, ok := as[s]
+		if target == nil && !ok {
+			unmatched = append(unmatched, s)
+			continue
+		}
 
-		if !ok && base == nil {
-			newSymbols = append(newSymbols, s)
+		if target == nil {
+			target = match
+			target.Symbols = append(target.Symbols, unmatched...)
+			unmatched = nil
+			for _, matchedSymbol := range match.Symbols {
+				processed[matchedSymbol] = true
+			}
 			continue
 		}
 
 		if !ok {
-			base.Symbols = append(base.Symbols, s)
-			as[s] = match
+			target.Symbols = append(target.Symbols, s)
+			processed[s] = true
 			continue
 		}
 
-		if base == nil {
-			base = match
-			for _, s := range newSymbols {
-				base.Symbols = append(base.Symbols, s)
-			}
+		if match == target {
 			continue
 		}
 
-		if base != match {
-			// Resolve conflict
-			mergeAsset(*match, base)
-			continue
+		for _, matchedSymbol := range match.Symbols {
+			processed[matchedSymbol] = true
 		}
+		mergeAsset(*match, target)
 	}
 
-	if base == nil {
-		base = &AssetImport{Instrument: Instrument{Symbols: ss}}
+	// If still no matches found
+	if target == nil {
+		target = &AssetImport{Instrument: Instrument{Symbols: ss}}
 	}
 
-	for _, s := range base.Symbols {
-		as[s] = base
+	for _, s := range target.Symbols {
+		as[s] = target
 	}
 
-	return base
+	return target
 }
 
 // mergeAsset merges information on the assets and jions all founc events: trades, dividends and withholding tax tax
@@ -199,12 +244,13 @@ func mergeAsset(src AssetImport, tgt *AssetImport) {
 	tgt.Symbols = list
 
 	// TODO improve and check for mismatch when not empty
-	if tgt.Category == "" {
+	if tgt.Category == "" && src.Category != "" {
 		tgt.Category = src.Category
 	}
 
 	tgt.Trades = append(tgt.Trades, src.Trades...)
 	tgt.Dividends = append(tgt.Dividends, src.Dividends...)
+	tgt.WithholdingTax = append(tgt.WithholdingTax, src.WithholdingTax...)
 }
 
 // assets maps imported asset information by symbol, for easier lookup while importing
@@ -221,7 +267,6 @@ func (a assets) list() []AssetImport {
 		list = append(list, *a)
 		listed[a] = true
 	}
-
 	return list
 }