Merge branch 'master' of github.com:diadata-org/diadata

cmd/assetCollectionService/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/assetCollectionService
 go 1.17
 
 require (
-	github.com/diadata-org/diadata v1.4.340
+	github.com/diadata-org/diadata v1.4.357
 	github.com/sirupsen/logrus v1.8.1
 )
 

cmd/assetCollectionService/main.go

@@ -132,6 +132,14 @@ func NewAssetScraper(exchange string, secret string) source.AssetSource {
 		return source.NewOsmosisScraper(exchanges[dia.OsmosisExchange])
 	case dia.VelodromeExchange:
 		return source.NewVelodromeAssetSource(exchanges[dia.VelodromeExchange])
+	case dia.TraderJoeExchangeV2_1:
+		return source.NewTraderJoeAssetSource(exchanges[dia.TraderJoeExchangeV2_1])
+	case dia.TraderJoeExchangeV2_1Arbitrum:
+		return source.NewTraderJoeAssetSource(exchanges[dia.TraderJoeExchangeV2_1Arbitrum])
+	case dia.TraderJoeExchangeV2_1Avalanche:
+		return source.NewTraderJoeAssetSource(exchanges[dia.TraderJoeExchangeV2_1Avalanche])
+	case dia.TraderJoeExchangeV2_1BNB:
+		return source.NewTraderJoeAssetSource(exchanges[dia.TraderJoeExchangeV2_1BNB])
 	case "assetlists":
 		return source.NewJSONReader(exchange, secret)
 	default:

cmd/exchange-scrapers/collector/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/exchange-scrapers/collector
 go 1.17
 
 require (
-	github.com/diadata-org/diadata v1.4.343
+	github.com/diadata-org/diadata v1.4.359
 	github.com/segmentio/kafka-go v0.4.35
 	github.com/sirupsen/logrus v1.9.0
 )

cmd/http/restServer/go.mod

@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/appleboy/gin-jwt/v2 v2.6.4
-	github.com/diadata-org/diadata v1.4.314
+	github.com/diadata-org/diadata v1.4.356
 	github.com/gin-contrib/cache v1.1.0
 	github.com/gin-gonic/contrib v0.0.0-20201101042839-6a891bf89f19
 	github.com/gin-gonic/gin v1.8.1

cmd/liquidityScraper/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/liquidityScraper
 go 1.17
 
 require (
-	github.com/diadata-org/diadata v1.4.340
+	github.com/diadata-org/diadata v1.4.357
 	github.com/sirupsen/logrus v1.8.1
 )
 

cmd/services/tradesBlockService/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/services/tradesBlockService
 go 1.17
 
 require (
-	github.com/diadata-org/diadata v1.4.324
+	github.com/diadata-org/diadata v1.4.359
 	github.com/segmentio/kafka-go v0.4.35
 	github.com/sirupsen/logrus v1.8.1
 )

config/MEXC.json

@@ -414,6 +414,12 @@
       "Exchange": "MEXC",
       "Ignore": false
     },
+    {
+      "Symbol": "PENDULUM",
+      "ForeignName": "PENDULUMUSDT",
+      "Exchange": "MEXC",
+      "Ignore": false
+    },
     {
       "Symbol": "PEPE",
       "ForeignName": "PEPEUSDT",

config/gitcoinverified/MEXC.json

@@ -378,6 +378,12 @@
       "Blockchain": "Ethereum",
       "Address": "0x45804880De22913dAFE09f4980848ECE6EcbAf78"
     },
+    {
+      "Symbol": "PENDULUM",
+      "Exchange": "MEXC",
+      "Blockchain": "Pendulum",
+      "Address": "0x0000000000000000000000000000000000000000"
+    },
     {
       "Symbol": "PEPE",
       "Exchange": "MEXC",

internal/pkg/tradesBlockService/tradesBlockService.go

@@ -581,7 +581,7 @@ func buildBridge(t dia.Trade) dia.Asset {
 			}
 		}
 	}
-	if basetoken.Blockchain == dia.ARBITRUM && (t.Source == dia.UniswapExchangeV3Arbitrum || t.Source == dia.SushiSwapExchangeArbitrum || t.Source == dia.CamelotExchange) {
+	if basetoken.Blockchain == dia.ARBITRUM && (t.Source == dia.UniswapExchangeV3Arbitrum || t.Source == dia.SushiSwapExchangeArbitrum || t.Source == dia.CamelotExchange || t.Source == dia.TraderJoeExchangeV2_1Arbitrum) {
 		if basetoken.Address == common.HexToAddress("0x82aF49447D8a07e3bd95BD0d56f35241523fBab1").Hex() {
 			basetoken = dia.Asset{
 				Symbol:     "ETH",

pkg/dia/scraper/exchange-scrapers/APIScraper.go

@@ -243,6 +243,14 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD
 		return NewUniswapScraper(Exchanges[dia.StellaswapExchange], scrape, relDB)
 	case dia.WanswapExchange:
 		return NewUniswapScraper(Exchanges[dia.WanswapExchange], scrape, relDB)
+	case dia.TraderJoeExchangeV2_1:
+		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1], scrape, relDB)
+	case dia.TraderJoeExchangeV2_1Arbitrum:
+		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1Arbitrum], scrape, relDB)
+	case dia.TraderJoeExchangeV2_1Avalanche:
+		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1Avalanche], scrape, relDB)
+	case dia.TraderJoeExchangeV2_1BNB:
+		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1BNB], scrape, relDB)
 	case dia.OsmosisExchange:
 		return NewOsmosisScraper(Exchanges[dia.OsmosisExchange], scrape, relDB)
 	case dia.ZenlinkswapExchange:

pkg/dia/scraper/exchange-scrapers/TraderJoeScraper.go

@@ -72,8 +72,6 @@ type TraderJoeScraper struct {
 	pairReceived chan *TraderJoePair
 	// Name of the exchange.
 	exchangeName string
-	// Ethereum block number to start scraping from.
-	startBlock uint64
 	// Time interval for waiting between actions.
 	waitTime int
 	// Option to listen for trading pairs by address.
@@ -102,16 +100,15 @@ func NewTraderJoeScraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB
 		log.Fatal("parse LISTEN_BY_ADDRESS: ", err)
 	}
 
-	// TODO: revisit block numbers
 	switch exchange.Name {
 	case dia.TraderJoeExchangeV2_1:
-		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200", uint64(22757913))
+		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200")
 	case dia.TraderJoeExchangeV2_1Arbitrum:
-		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200", uint64(107808631)) // startBlock pulled from: https://arbiscan.io/tx/0x7e1d7980bdbfb09b0a64d35e660cee300accaf017674db7a3ac9f655ca72ac35
+		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200")
 	case dia.TraderJoeExchangeV2_1Avalanche:
-		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200", uint64(165))
+		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200")
 	case dia.TraderJoeExchangeV2_1BNB:
-		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200", uint64(27099340)) // startBlock pulled from: https://bscscan.com/tx/0x4c8a1f2f3d1a92281a3067aaea799536118e02a072380ccd3df642e8adecba6d
+		tjs = makeTraderJoeScraper(exchange, listenByAddress, "", "", "200")
 	}
 
 	tjs.relDB = relDB
@@ -142,7 +139,7 @@ func NewTraderJoeScraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB
 // makeTraderJoeScraper creates and initializes a Trader Joe scraper instance with the given exchange information,
 // connection details, and configuration parameters. It establishes REST and WebSocket clients for the blockchain,
 // determines wait time, and sets up various channels and data structures for scraping tasks.
-func makeTraderJoeScraper(exchange dia.Exchange, listenByAddress bool, restDial string, wsDial string, waitMilliseconds string, startBlock uint64) *TraderJoeScraper {
+func makeTraderJoeScraper(exchange dia.Exchange, listenByAddress bool, restDial string, wsDial string, waitMilliseconds string) *TraderJoeScraper {
 	var (
 		restClient, wsClient *ethclient.Client
 		s                    *TraderJoeScraper
@@ -179,7 +176,6 @@ func makeTraderJoeScraper(exchange dia.Exchange, listenByAddress bool, restDial
 		chanTrades:             make(chan *dia.Trade),
 		waitTime:               waitTime,
 		listenByAddress:        listenByAddress,
-		startBlock:             startBlock,
 		factoryContractAddress: common.HexToAddress(exchange.Contract),
 	}
 
@@ -203,19 +199,31 @@ func (tjs *TraderJoeScraper) GetSwapsChannel(pairAddress common.Address) (chan *
 }
 
 func (tjs *TraderJoeScraper) normalizeTraderJoeSwap(swap traderjoeILBPair.ILBPairSwap) (normalizedSwap TraderJoeSwap) {
+
 	pair := MapOfPools[swap.Raw.Address.Hex()]
 	decimals0 := int(pair.Token0.Decimals)
 	decimals1 := int(pair.Token1.Decimals)
 
-	amountIn := new(big.Int).SetBytes(swap.AmountsIn[:])
-	amountOut := new(big.Int).SetBytes(swap.AmountsOut[:])
+	amount1In := new(big.Int).SetBytes(swap.AmountsIn[:16])
+	amount0In := new(big.Int).SetBytes(swap.AmountsIn[16:])
+	amount1Out := new(big.Int).SetBytes(swap.AmountsOut[:16])
+	amount0Out := new(big.Int).SetBytes(swap.AmountsOut[16:])
+	var amount0, amount1 float64
 
-	amount0, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(amountIn), new(big.Float).SetFloat64(math.Pow10(decimals0))).Float64()
-	amount1, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(amountOut), new(big.Float).SetFloat64(math.Pow10(decimals1))).Float64()
+	if amount0In.Cmp(big.NewInt(0)) == 1 {
+		amount0, _ = new(big.Float).Quo(big.NewFloat(0).SetInt(amount0In), new(big.Float).SetFloat64(math.Pow10(decimals0))).Float64()
+	} else {
+		amount0, _ = new(big.Float).Quo(big.NewFloat(0).SetInt(amount0Out), new(big.Float).SetFloat64(math.Pow10(decimals0))).Float64()
+	}
+	if amount1In.Cmp(big.NewInt(0)) == 1 {
+		amount1, _ = new(big.Float).Quo(big.NewFloat(0).SetInt(amount1In), new(big.Float).SetFloat64(math.Pow10(decimals1))).Float64()
+	} else {
+		amount1, _ = new(big.Float).Quo(big.NewFloat(0).SetInt(amount1Out), new(big.Float).SetFloat64(math.Pow10(decimals1))).Float64()
+	}
 
 	normalizedSwap = TraderJoeSwap{
 		ID:        swap.Raw.TxHash.Hex(),
-		Timestamp: time.Now().Unix(),
+		Timestamp: time.Now().UnixNano(),
 		Pair:      pair,
 		Amount0:   amount0,
 		Amount1:   amount1,
@@ -383,11 +391,11 @@ func (tjs *TraderJoeScraper) sendTrade(traderjoeswap TraderJoeSwap, pool *Trader
 	t := &dia.Trade{
 		Symbol:         pool.Token0.Symbol,
 		Pair:           pool.ForeignName,
-		QuoteToken:     token1,
-		BaseToken:      token0,
+		QuoteToken:     token0,
+		BaseToken:      token1,
 		Price:          price,
 		Volume:         volume,
-		Time:           time.Unix(traderjoeswap.Timestamp, 0),
+		Time:           time.Unix(0, traderjoeswap.Timestamp),
 		PoolAddress:    pool.Address.Hex(),
 		ForeignTradeID: traderjoeswap.ID,
 		//EstimatedUSDPrice: 0,
@@ -422,6 +430,18 @@ type TraderJoeTradeScraper struct {
 	pair   dia.ExchangePair
 }
 
+func (tjs *TraderJoeScraper) FetchAvailablePairs() ([]dia.ExchangePair, error) {
+	return []dia.ExchangePair{}, nil
+}
+
+func (tjs *TraderJoeScraper) FillSymbolData(symbol string) (dia.Asset, error) {
+	return dia.Asset{Symbol: symbol}, nil
+}
+
+func (tjs *TraderJoeScraper) NormalizePair(pair dia.ExchangePair) (dia.ExchangePair, error) {
+	return pair, nil
+}
+
 // Close closes any existing API connections, as well as channels of PairScrapers from calls to ScrapePair
 func (tjs *TraderJoeScraper) Close() error {
 	if tjs.closed {

pkg/dia/scraper/liquidity-scrapers/ScraperInterface.go

@@ -65,16 +65,6 @@ func NewLiquidityScraper(source string, relDB *models.RelDB, datastore *models.D
 		return NewUniswapScraper(exchanges[dia.NetswapExchange], relDB, datastore)
 	case dia.HuckleberryExchange:
 		return NewUniswapScraper(exchanges[dia.HuckleberryExchange], relDB, datastore)
-	case dia.TraderJoeExchange:
-		return NewUniswapScraper(exchanges[dia.TraderJoeExchange], relDB, datastore)
-	case dia.TraderJoeExchangeV2_1:
-		return NewUniswapScraper(exchanges[dia.TraderJoeExchangeV2_1], relDB, datastore)
-	case dia.TraderJoeExchangeV2_1Arbitrum:
-		return NewUniswapScraper(exchanges[dia.TraderJoeExchangeV2_1Arbitrum], relDB, datastore)
-	case dia.TraderJoeExchangeV2_1Avalanche:
-		return NewUniswapScraper(exchanges[dia.TraderJoeExchangeV2_1Avalanche], relDB, datastore)
-	case dia.TraderJoeExchangeV2_1BNB:
-		return NewUniswapScraper(exchanges[dia.TraderJoeExchangeV2_1BNB], relDB, datastore)
 	case dia.PangolinExchange:
 		return NewUniswapScraper(exchanges[dia.PangolinExchange], relDB, datastore)
 	case dia.TethysExchange:
@@ -96,6 +86,17 @@ func NewLiquidityScraper(source string, relDB *models.RelDB, datastore *models.D
 	case dia.WanswapExchange:
 		return NewUniswapScraper(exchanges[dia.WanswapExchange], relDB, datastore)
 
+	case dia.TraderJoeExchange:
+		return NewTraderJoeLiquidityScraper(exchanges[dia.TraderJoeExchange], relDB, datastore)
+	case dia.TraderJoeExchangeV2_1:
+		return NewTraderJoeLiquidityScraper(exchanges[dia.TraderJoeExchangeV2_1], relDB, datastore)
+	case dia.TraderJoeExchangeV2_1Arbitrum:
+		return NewTraderJoeLiquidityScraper(exchanges[dia.TraderJoeExchangeV2_1Arbitrum], relDB, datastore)
+	case dia.TraderJoeExchangeV2_1Avalanche:
+		return NewTraderJoeLiquidityScraper(exchanges[dia.TraderJoeExchangeV2_1Avalanche], relDB, datastore)
+	case dia.TraderJoeExchangeV2_1BNB:
+		return NewTraderJoeLiquidityScraper(exchanges[dia.TraderJoeExchangeV2_1BNB], relDB, datastore)
+
 	case dia.CurveFIExchange:
 		return NewCurveFIScraper(exchanges[dia.CurveFIExchange], relDB, datastore)
 	case dia.CurveFIExchangePolygon:

pkg/dia/scraper/liquidity-scrapers/TraderJoeScraper.go

@@ -10,6 +10,7 @@ import (
 	"github.com/diadata-org/diadata/pkg/dia"
 	"github.com/diadata-org/diadata/pkg/dia/helpers/ethhelper"
 	traderjoe "github.com/diadata-org/diadata/pkg/dia/scraper/exchange-scrapers/traderjoe2.1"
+	"github.com/diadata-org/diadata/pkg/dia/scraper/exchange-scrapers/traderjoe2.1/traderjoeILBPair"
 	models "github.com/diadata-org/diadata/pkg/model"
 	"github.com/diadata-org/diadata/pkg/utils"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -43,13 +44,13 @@ func NewTraderJoeLiquidityScraper(exchange dia.Exchange, relDB *models.RelDB, da
 
 	switch exchange.Name {
 	case dia.TraderJoeExchangeV2_1:
-		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(12369621))
+		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(17821282))
+	case dia.TraderJoeExchangeV2_1Arbitrum:
+		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(77473199))
 	case dia.TraderJoeExchangeV2_1Avalanche:
-		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(22757913))
+		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(28371397))
 	case dia.TraderJoeExchangeV2_1BNB:
-		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(165))
-	case dia.TraderJoeExchangeV2_1Arbitrum:
-		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(26956207))
+		tjls = makeTraderJoeScraper(exchange, "", "", relDB, datastore, "200", uint64(27099340))
 	}
 
 	go func() {
@@ -60,7 +61,7 @@ func NewTraderJoeLiquidityScraper(exchange dia.Exchange, relDB *models.RelDB, da
 
 // makeTraderJoeScraper initializes a Trader Joe liquidity scraper, creating an instance of the
 // TraderJoeLiquidityScraper struct with the specified configuration and parameters.
-func makeTraderJoeScraper(exchange dia.Exchange, restDial string, websocketDial string, relDB *models.RelDB, datastore *models.DB, waitMilliSeconds string, startBlock uint64) *TraderJoeLiquidityScraper {
+func makeTraderJoeScraper(exchange dia.Exchange, restDial string, wsDial string, relDB *models.RelDB, datastore *models.DB, waitMilliSeconds string, startBlock uint64) *TraderJoeLiquidityScraper {
 	var (
 		restClient  *ethclient.Client
 		wsClient    *ethclient.Client
@@ -75,7 +76,7 @@ func makeTraderJoeScraper(exchange dia.Exchange, restDial string, websocketDial
 	if err != nil {
 		log.Fatal("init rest client: ", err)
 	}
-	wsClient, err = ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_WS", websocketDial))
+	wsClient, err = ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_WS", wsDial))
 	if err != nil {
 		log.Fatal("init ws client: ", err)
 	}
@@ -107,12 +108,7 @@ func makeTraderJoeScraper(exchange dia.Exchange, restDial string, websocketDial
 // fetchPools retrieves pool creation events from the Trader Joe factory contract address and processes them.
 func (tjls *TraderJoeLiquidityScraper) fetchPools() {
 
-	// filter from contract created at: https://etherscan.io/tx/0x8e42f2F4101563bF679975178e880FD87d3eFd4e
-
-	// Log a message to indicate the start of pool creation event retrieval.
 	log.Info("Fetching Trader Joe LBPairCreated events...")
-
-	// Log the factory contract address being used for filtering.
 	log.Info("Getting lb pairs creations from address: ", tjls.factoryContract)
 
 	// Initialize a count for the number of pairs processed.
@@ -172,35 +168,18 @@ func (tjls *TraderJoeLiquidityScraper) fetchPools() {
 		pool.Blockchain = dia.BlockChain{Name: tjls.blockchain}
 		pool.Address = lbPairCreated.Event.LBPair.Hex()
 
-		var (
-			balance0Big *big.Int
-			balance1Big *big.Int
-		)
-
-		// Create a token caller to interact with the liquidity pool's contract.
-		tokenCaller, err := ethhelper.NewTokenCaller(common.HexToAddress(lbPairCreated.Event.LBPair.Hex()), tjls.RestClient)
+		pairFiltererContract, err := traderjoeILBPair.NewILBPairCaller(lbPairCreated.Event.LBPair, tjls.RestClient)
 		if err != nil {
-			log.Warn("unable to create token caller", err)
-			balance0Big, balance1Big = big.NewInt(0), big.NewInt(0)
+			log.Fatal(err)
 		}
-
-		// Retrieve the token reserves from the liquidity pool contract.
-		if tokenCaller != nil {
-			var reserves []interface{}
-			if err = tokenCaller.Contract.Call(&bind.CallOpts{}, &reserves, "getReserves"); err != nil {
-				log.Warn("unable to get reserves", err)
-			}
-
-			if len(reserves) == 2 {
-				balance0Big, balance1Big = reserves[0].(*big.Int), reserves[1].(*big.Int)
-			} else {
-				balance0Big, balance1Big = big.NewInt(0), big.NewInt(0)
-			}
+		reserves, err := pairFiltererContract.GetReserves(&bind.CallOpts{})
+		if err != nil {
+			log.Fatal("get reserves on pool ", lbPairCreated.Event.LBPair.Hex())
 		}
 
 		// Calculate token balances in floating-point format.
-		balance0, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(balance0Big), new(big.Float).SetFloat64(math.Pow10(int(asset0.Decimals)))).Float64()
-		balance1, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(balance1Big), new(big.Float).SetFloat64(math.Pow10(int(asset1.Decimals)))).Float64()
+		balance0, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(reserves.ReserveX), new(big.Float).SetFloat64(math.Pow10(int(asset0.Decimals)))).Float64()
+		balance1, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(reserves.ReserveY), new(big.Float).SetFloat64(math.Pow10(int(asset1.Decimals)))).Float64()
 
 		// Append asset volumes to the pool.
 		pool.Assetvolumes = append(pool.Assetvolumes, dia.AssetVolume{Asset: asset0, Volume: balance0, Index: uint8(0)})