Overall: Fix build and test issues, dependencies

.travis.yml

@@ -2,11 +2,9 @@ dist: xenial
 
 language: go
 
-go:
-- "1.10.1"
-
 before_install:
-  - sudo apt-get install -y libsodium-dev
+  - sudo apt-get install -y pkg-config
+  - wget https://download.libsodium.org/libsodium/releases/old/unsupported/libsodium-1.0.12.tar.gz && tar -xf libsodium-1.0.12.tar.gz && cd libsodium-1.0.12 && ./configure && make && sudo make install && cd ../ && rm -rf libsodium-1.0.12 libsodium-1.0.12.tar.gz
 addons:
   apt:
     update: true

internal/pkg/blockchain-scrapers/blockchains/tezos/scrapers/xtz/xtz.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/DefinitelyNotAGoat/goTezos"
+	goTezos "github.com/DefinitelyNotAGoat/go-Tezos"
 	"github.com/diadata-org/diadata/pkg/dia"
 	log "github.com/sirupsen/logrus"
 	"os"
@@ -11,7 +11,8 @@ import (
 	"time"
 )
 
-const rpcURL = "http://tezosnode:8732"
+const rpcURL = "http://tezosnode"
+const rpcPort = ":8732"
 const xtzAddress = "tz1RCFbB9GpALpsZtu6J58sb74dm8qe6XBzv"
 const totalSupply = 763306930.69
 const decimals = 1e8
@@ -22,13 +23,13 @@ const blockFrequency = 60
 
 var client *dia.Client
 
-func retrieveXTZSupply() {
-	b, eg := goTezos.GetChainHead()
+func retrieveXTZSupply(gt *goTezos.GoTezos) {
+	b, eg := gt.GetChainHead()
 	if eg != nil {
 		log.Error("Error getting block:" + eg.Error())
 	} else {
 		log.Println("Block hash:" + b.Hash)
-		v, e := goTezos.GetAccountBalanceAtBlock(xtzAddress, b.Hash)
+		v, e := gt.GetAccountBalanceAtBlock(xtzAddress, b.Hash)
 		if e != nil {
 			log.Error("Error retrieving balance for:" + xtzAddress + " error:" + e.Error())
 		} else {
@@ -50,13 +51,13 @@ type Task struct {
 	ticker *time.Ticker
 }
 
-func (t *Task) run() {
+func (t *Task) run(gt *goTezos.GoTezos) {
 	for {
 		select {
 		case <-t.closed:
 			return
 		case <-t.ticker.C:
-			retrieveXTZSupply()
+			retrieveXTZSupply(gt)
 		}
 	}
 }
@@ -69,7 +70,8 @@ func (t *Task) stop() {
 }
 
 func main() {
-	goTezos.SetRPCURL(rpcURL)
+	gt := goTezos.NewGoTezos()
+	gt.AddNewClient(goTezos.NewTezosRPCClient(rpcURL, rpcPort))
 	task := &Task{
 		closed: make(chan struct{}),
 		ticker: time.NewTicker(time.Second * blockFrequency),
@@ -87,7 +89,7 @@ func main() {
 	c := make(chan os.Signal)
 	signal.Notify(c, os.Interrupt)
 	task.wg.Add(1)
-	go func() { defer task.wg.Done(); task.run() }()
+	go func() { defer task.wg.Done(); task.run(gt) }()
 	select {
 	case <-c:
 		log.Println("Got signal.")

internal/pkg/exchange-scrapers/CoinBaseScraper.go

@@ -6,7 +6,7 @@ import (
 	"github.com/diadata-org/diadata/pkg/dia"
 	"github.com/diadata-org/diadata/pkg/dia/helpers"
 	ws "github.com/gorilla/websocket"
-	"github.com/preichenberger/go-gdax"
+	gdax "github.com/preichenberger/go-gdax"
 	log "github.com/sirupsen/logrus"
 	"io/ioutil"
 	"net/http"
@@ -70,23 +70,23 @@ func (s *CoinBaseScraper) mainLoop() {
 			break
 		}
 		if message.Type == ChannelTicker {
-			ps, ok := s.pairScrapers[message.ProductId]
+			ps, ok := s.pairScrapers[message.ProductID]
 			if ok {
 				f64Price, err := strconv.ParseFloat(message.Price, 64)
 				if err == nil {
 					f64Volume, err := strconv.ParseFloat(message.LastSize, 64)
 					if err == nil {
-						if message.TradeId != 0 {
+						if message.TradeID != 0 {
 							if message.Side == "sell" {
 								f64Volume = -f64Volume
 							}
 							t := &dia.Trade{
 								Symbol:         ps.pair.Symbol,
-								Pair:           message.ProductId,
+								Pair:           message.ProductID,
 								Price:          f64Price,
 								Volume:         f64Volume,
 								Time:           message.Time.Time(),
-								ForeignTradeID: strconv.FormatInt(int64(message.TradeId), 16),
+								ForeignTradeID: strconv.FormatInt(int64(message.TradeID), 16),
 								Source:         s.exchangeName,
 							}
 							ps.parent.chanTrades <- t
@@ -98,7 +98,7 @@ func (s *CoinBaseScraper) mainLoop() {
 					log.Error("error parsing price " + message.Price)
 				}
 			} else {
-				log.Error("unknown productError" + message.ProductId)
+				log.Error("unknown productError" + message.ProductID)
 			}
 		}
 	}
@@ -155,11 +155,11 @@ func (s *CoinBaseScraper) FetchAvailablePairs() (pairs []dia.Pair, err error) {
 	err = json.Unmarshal(data, &ar)
 	if err == nil {
 		for _, p := range ar {
-			symbol, serr := s.normalizeSymbol(p.Id)
+			symbol, serr := s.normalizeSymbol(p.ID)
 			if serr == nil {
 				pairs = append(pairs, dia.Pair{
 					Symbol:      symbol,
-					ForeignName: p.Id,
+					ForeignName: p.ID,
 					Exchange:    s.exchangeName,
 				})
 			} else {

internal/pkg/exchange-scrapers/OKExScraper.go

@@ -162,11 +162,11 @@ func (s *OKExScraper) mainLoop() {
 								ps.parent.chanTrades <- t
 
 							} else {
-								log.Error("parsing volume %v", f64Volume_string)
+								log.Errorf("parsing volume %v", f64Volume_string)
 							}
 
 						} else {
-							log.Error("parsing price %v", f64Price_string)
+							log.Errorf("parsing price %v", f64Price_string)
 						}
 					}
 				}

internal/pkg/graphService/graph.go

@@ -77,8 +77,7 @@ func PriceGraph(prices []float64, times []int64, path string) error {
 	graph.BackgroundColor = color.Transparent
 	line.LineStyle.Color = blue
 
-	line.ShadeColor = new(color.Color)
-	*line.ShadeColor = lightBlue
+	line.FillColor = lightBlue
 
 	// add a small margin at the bottom
 	graph.Y.Min = min - (max-min)*0.1

internal/pkg/tradesBlockService/tradesBlockService.go

@@ -110,7 +110,7 @@ func (s *TradesBlockService) process(t dia.Trade) {
 
 	if s.currentBlock != nil &&
 		s.currentBlock.TradesBlockData.BeginTime.After(t.Time) {
-		log.Debug("ignore trade should be in previous block %v", t)
+		log.Debugf("ignore trade should be in previous block %v", t)
 		ignoreTrade = true
 	}
 
@@ -136,7 +136,7 @@ func (s *TradesBlockService) process(t dia.Trade) {
 		}
 		s.currentBlock.TradesBlockData.Trades = append(s.currentBlock.TradesBlockData.Trades, t)
 	} else {
-		log.Debug("ignore trade  %v", t)
+		log.Debugf("ignore trade  %v", t)
 	}
 }
 

pkg/dia/ApiClient.go

@@ -110,7 +110,7 @@ func GetSupply(symbol string) (*Supply, error) {
 		if err != nil {
 			return nil, err
 		}
-		log.Debug("%s\n", string(contents))
+		log.Debugf("%s\n", string(contents))
 		var b Supply
 		err = b.UnmarshalBinary(contents)
 		if err == nil {
@@ -136,7 +136,7 @@ func GetSymbolsList(url string) ([]string, error) {
 		if err != nil {
 			return nil, err
 		}
-		log.Debug("%s\n", string(contents))
+		log.Debugf("%s\n", string(contents))
 		var b Symbols
 
 		err = json.Unmarshal(contents, &b)

pkg/dia/helpers/kafkaHelper/Kafka.go

@@ -184,7 +184,7 @@ func NewReaderXElementsBeforeLastMessage(topic int, x int64) *kafka.Reader {
 	if err == nil && o-x > 0 {
 		offset = o - x
 	} else {
-		log.Warning("err %v on readOffset on topic %v", err, topic)
+		log.Warningf("err %v on readOffset on topic %v", err, topic)
 	}
 
 	log.Println("NewReaderXElementsBeforeLastMessage: setting offset ", offset, "/", o)

pkg/model/db.go

@@ -286,12 +286,12 @@ func (db *DB) setZSETValue(key string, value float64, unixTime int64, maxWindow
 	}).Err()
 	log.Debug("SetZSETValue ", key, member, unixTime)
 	if err != nil {
-		log.Error("Error: %v on SetZSETValue %v\n", err, key)
+		log.Errorf("Error: %v on SetZSETValue %v\n", err, key)
 	}
 	// purging old values
 	err = db.redisClient.ZRemRangeByScore(key, "-inf", "("+strconv.FormatInt(unixTime-maxWindow, 10)).Err()
 	if err != nil {
-		log.Error("Error: %v on SetZSETValue %v\n", err, key)
+		log.Errorf("Error: %v on SetZSETValue %v\n", err, key)
 	}
 
 	if err := db.redisClient.Expire(key, TimeOutRedis).Err(); err != nil {
@@ -312,7 +312,7 @@ func (db *DB) getZSETValue(key string, atUnixTime int64) (float64, error) {
 	if err == nil {
 		if len(vals) > 0 {
 			fmt.Sscanf(vals[len(vals)-1].Member.(string), "%f", &result)
-			log.Debug("returned value: %v", result)
+			log.Debugf("returned value: %v", result)
 		} else {
 			err = errors.New("getZSETValue no value found")
 		}
@@ -322,11 +322,11 @@ func (db *DB) getZSETValue(key string, atUnixTime int64) (float64, error) {
 
 func (db *DB) getZSETSum(key string) (*float64, error) {
 
-	log.Debug("getZSETSum: %v \n", key)
+	log.Debugf("getZSETSum: %v \n", key)
 
 	vals, err := db.redisClient.ZRange(key, 0, -1).Result()
 	if err != nil {
-		log.Error("Error: %v on getZSETSum %v\n", err, key)
+		log.Errorf("Error: %v on getZSETSum %v\n", err, key)
 		return nil, err
 	} else {
 		result := 0.0
@@ -347,7 +347,7 @@ func (db *DB) getZSETLastValue(key string) (float64, int64, error) {
 	if err == nil {
 		if len(vals) == 1 {
 			fmt.Sscanf(vals[0], "%f %d", &value, &unixTime)
-			log.Debug("returned value: %v", value)
+			log.Debugf("returned value: %v", value)
 		} else {
 			err = errors.New("getZSETLastValue no value found")
 			log.Errorln("Error: on getZSETLastValue", err, key)

pkg/model/exchanges.go

@@ -69,7 +69,7 @@ func (db *DB) GetExchangesForSymbol(symbol string) ([]string, error) { // TOFIX.
 			}
 		}
 		if cursor == 0 {
-			log.Debug("GetExchangesForSymbol %v returns %v", key, result)
+			log.Debugf("GetExchangesForSymbol %v returns %v", key, result)
 			return result, nil
 		}
 	}
@@ -88,7 +88,7 @@ func (db *DB) GetAvailablePairsForExchange(exchange string) ([]dia.Pair, error)
 	p := dia.Pairs{}
 	err := db.redisClient.Get(key).Scan(&p)
 	if err != nil {
-		log.Error("Error: %v on GetAvailablePairsForExchange %v\n", err, exchange)
+		log.Errorf("Error: %v on GetAvailablePairsForExchange %v\n", err, exchange)
 		return nil, err
 	}
 	return p, nil

pkg/model/pairs.go

@@ -30,7 +30,7 @@ func (db *DB) GetPairs(exchange string) ([]dia.Pair, error) {
 			}
 		}
 		if cursor == 0 {
-			log.Debug("GetPairs %v returns %v", key, result)
+			log.Debugf("GetPairs %v returns %v", key, result)
 			return result, nil
 		}
 	}

pkg/model/quotation.go

@@ -65,7 +65,7 @@ func (db *DB) GetPriceUSD(symbol string) (float64, error) {
 	err := db.redisClient.Get(key).Scan(value)
 	if err != nil {
 		if err != redis.Nil {
-			log.Error("Error: %v on GetPriceUSD %v\n", err, symbol)
+			log.Errorf("Error: %v on GetPriceUSD %v\n", err, symbol)
 		}
 		return 0.0, err
 	}
@@ -78,7 +78,7 @@ func (db *DB) GetQuotation(symbol string) (*Quotation, error) {
 	err := db.redisClient.Get(key).Scan(value)
 	if err != nil {
 		if err != redis.Nil {
-			log.Error("Error: %v on GetQuotation %v\n", err, key)
+			log.Errorf("Error: %v on GetQuotation %v\n", err, key)
 		}
 		return nil, err
 	}

pkg/model/supplies.go

@@ -28,7 +28,7 @@ func (db *DB) SymbolsWithASupply() ([]string, error) {
 			result = append(result, strings.Replace(value, key, "", 1))
 		}
 		if cursor == 0 {
-			log.Debug("SymbolsWithASupply %v returns %v", key, result)
+			log.Debugf("SymbolsWithASupply %v returns %v", key, result)
 			return result, nil
 		}
 	}
@@ -49,7 +49,7 @@ func (a *DB) GetSupply(symbol string) (*dia.Supply, error) {
 		value := &dia.Supply{}
 		err := a.redisClient.Get(key).Scan(value)
 		if err != nil {
-			log.Error("Error: %v on GetSupply %v\n", err, symbol)
+			log.Errorf("Error: %v on GetSupply %v\n", err, symbol)
 			return nil, err
 		}
 		return value, err
@@ -61,7 +61,7 @@ func (a *DB) SetSupply(supply *dia.Supply) error {
 	log.Debug("setting ", key, supply)
 	err := a.redisClient.Set(key, supply, 0).Err()
 	if err != nil {
-		log.Error("Error: %v on SetSupply %v\n", err, supply.Symbol)
+		log.Errorf("Error: %v on SetSupply %v\n", err, supply.Symbol)
 	}
 	return err
 }

pkg/model/symbols.go

@@ -54,7 +54,7 @@ func (db *DB) GetSymbols(exchange string) ([]string, error) {
 			}
 		}
 		if cursor == 0 {
-			log.Debug("GetSymbols %v returns %v", key, result)
+			log.Debugf("GetSymbols %v returns %v", key, result)
 			return result, nil
 		}
 	}