diff --git a/liquidator/jest.config.ts b/liquidator/jest.config.ts index 3c0693a..68d9dcf 100644 --- a/liquidator/jest.config.ts +++ b/liquidator/jest.config.ts @@ -2,6 +2,7 @@ import type {Config} from 'jest'; const config: Config = { verbose: true, + testTimeout: 30000, }; export default config; diff --git a/liquidator/src/BaseExecutor.ts b/liquidator/src/BaseExecutor.ts index b5e551d..aa54bc5 100644 --- a/liquidator/src/BaseExecutor.ts +++ b/liquidator/src/BaseExecutor.ts @@ -3,15 +3,15 @@ import { Coin } from '@cosmjs/proto-signing' import { CosmWasmClient } from '@cosmjs/cosmwasm-stargate' import { RedisInterface } from './redis.js' import { AMMRouter } from './AmmRouter.js' -import fetch from 'cross-fetch' -import { Pagination, Pool } from './types/Pool.js' import 'dotenv/config.js' import { MarketInfo } from './rover/types/MarketInfo.js' import { CSVWriter, Row } from './CsvWriter.js' -import { camelCaseKeys } from './helpers.js' + import BigNumber from 'bignumber.js' import { fetchRedbankData } from './query/hive.js' import { PriceResponse } from 'marsjs-types/creditmanager/generated/mars-mock-oracle/MarsMockOracle.types.js' +import { PoolDataProviderInterface } from './amm/PoolDataProviderInterface.js' +import { AstroportPoolProvider } from './amm/AstroportPoolProvider.js' export interface BaseExecutorConfig { lcdEndpoint: string @@ -31,22 +31,12 @@ export interface BaseExecutorConfig { * @param config holds the neccessary configuration for the executor to operate */ export class BaseExecutor { - // Configuration should always be override by extending class - public config: BaseExecutorConfig - - // Helpers - public ammRouter: AMMRouter // Data public prices: Map = new Map() public balances: Map = new Map() public markets: MarketInfo[] = [] - // clients - public client: SigningStargateClient - public queryClient: CosmWasmClient - public redis: RedisInterface - // variables private poolsNextRefresh = 0 @@ -61,21 +51,22 @@ export class BaseExecutor { ]) constructor( - config: BaseExecutorConfig, - client: SigningStargateClient, - queryClient: CosmWasmClient, - ) { - this.config = config - this.ammRouter = new AMMRouter() - this.redis = new RedisInterface() - this.client = client - this.queryClient = queryClient - } + public config: BaseExecutorConfig, + private client: SigningStargateClient, + private queryClient: CosmWasmClient, + private poolProvider: PoolDataProviderInterface, + private redis : RedisInterface = new RedisInterface(), + public ammRouter : AMMRouter = new AMMRouter() + ) {} async initiateRedis(): Promise { await this.redis.connect(this.config.redisEndpoint) } + async initiateAstroportPoolProvider(): Promise { + await (this.poolProvider as AstroportPoolProvider).initiate() + } + applyAvailableLiquidity = (market: MarketInfo): MarketInfo => { // Available liquidity = deposits - borrows. However, we need to // compute the underlying uasset amounts from the scaled amounts. @@ -152,45 +143,9 @@ export class BaseExecutor { if (this.poolsNextRefresh < currentTime) { - const pools = await this.loadPools() + const pools = await this.poolProvider.loadPools() this.ammRouter.setPools(pools) this.poolsNextRefresh = Date.now() + this.config.poolsRefreshWindow } } - - loadPools = async (): Promise => { - let fetchedAllPools = false - let nextKey = '' - let pools: Pool[] = [] - let totalPoolCount = 0 - while (!fetchedAllPools) { - const response = await fetch( - `${this.config.lcdEndpoint}/osmosis/gamm/v1beta1/pools${nextKey}`, - ) - const responseJson: any = await response.json() - - if (responseJson.pagination === undefined) { - fetchedAllPools = true - return pools - } - - const pagination = camelCaseKeys(responseJson.pagination) as Pagination - - // osmosis lcd query returns total pool count as 0 after page 1 (but returns the correct count on page 1), so we need to only set it once - if (totalPoolCount === 0) { - totalPoolCount = pagination.total - } - - const poolsRaw = responseJson.pools as Pool[] - - poolsRaw.forEach((pool) => pools.push(camelCaseKeys(pool) as Pool)) - - nextKey = `?pagination.key=${pagination.nextKey}` - if (pools.length >= totalPoolCount) { - fetchedAllPools = true - } - } - - return pools - } } diff --git a/liquidator/src/amm/AstroportPoolProvider.ts b/liquidator/src/amm/AstroportPoolProvider.ts new file mode 100644 index 0000000..e51ed6f --- /dev/null +++ b/liquidator/src/amm/AstroportPoolProvider.ts @@ -0,0 +1,229 @@ +import { sleep } from "../helpers"; +import { Pool, PoolAsset } from "../types/Pool"; +import { PoolDataProviderInterface } from "./PoolDataProviderInterface"; +import fetch from 'cross-fetch' +import { Asset, AssetInfo, AssetInfoNative, ContractQueryPairs, ContractQueryPool, Pair, PoolResponseData, Query, ResponseData } from "./types/AstroportTypes"; + +export class AstroportPoolProvider implements PoolDataProviderInterface { + + private maxRetries = 8 + + private pairs : Pair[] = [] + + constructor( + private astroportFactory: string, + private graphqlEndpoint : string, + ) {} + + initiate = async () => { + this.pairs = await this.fetchPairContracts(this.astroportFactory) + + // refresh pool contracts every 30 minutes + setInterval(this.fetchPairContracts, 1000 * 60 * 30) + } + + setPairs = (pairs: Pair[]) => { + this.pairs = pairs + } + + getPairs = () : Pair[] => { + return this.pairs + } + + loadPools = async ():Promise => { + let retries = 0 + + if (this.pairs.length === 0) { + throw new Error("Pools still not loaded. Ensure you have called initiate()") + } + + const poolQueries = this.pairs.map((pair) => this.producePoolQuery(pair) ) + + + // execute the query + while (retries < this.maxRetries) { + try { + const response = await fetch(this.graphqlEndpoint, { + method: 'post', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(poolQueries), + }); + + const responseData : PoolResponseData[] =await response.json() + + return responseData.map((poolResponse, index) => { + + // Our address is our key. We add this in to our graphql queries so we can identify the correct pool + const address : string = Object.keys(poolResponse.data)[0] + + const queryData : ContractQueryPool = poolResponse.data[address]!.contractQuery + const pool : Pool = { + address : address, + id : index as unknown as Long, + poolAssets : this.producePoolAssets(queryData.assets), + swapFee : "0.00", + } + + return pool + }) + } catch(err) { + console.error(err) + retries += 1 + await sleep(1000) + } + } + + return [] + } + + async fetchPairContracts(contractAddress: string, limit = 10): Promise { + let startAfter = this.findLatestPair(this.pairs) + let retries = 0 + + const pairs : Pair[] = [] + + // Loop until we find all the assets or we hit our max retries + while (retries < this.maxRetries) { + try { + + const query = this.producePairQuery(startAfter, limit, contractAddress) + + const response = await fetch(this.graphqlEndpoint, { + method: 'post', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(query), + }); + + // get our data + const responseData : ResponseData = await response.json(); + const contractQueryData = responseData.data.wasm.contractQuery as ContractQueryPairs + + // Filter out pairs that are not XYK + const batchPairs = contractQueryData.pairs + .filter((pair) => pair.pair_type.xyk !== undefined) + + if (batchPairs.length === 0) { + // we have no more pairs to load + return pairs + } else { + + pairs.concat(batchPairs) + + let assets = pairs[pairs.length - 1].asset_infos + startAfter = `[ + ${this.produceStartAfterAsset(assets[0])}, + ${this.produceStartAfterAsset(assets[1])} + ]` + } + + } catch (error) { + console.error(error); + retries += 1 + await sleep(1000) + } + } + + return pairs + } + + private producePoolQuery = (pair : Pair) : Query => { + + const poolQuery = ` + query($contractAddress: String!){ + ${pair.contract_addr}:wasm { + contractQuery(contractAddress:$contractAddress, query: { + pool: {} + }) + } + } + ` + return { + query : poolQuery, + variables : { contractAddress : pair.contract_addr } + } + } + + private producePairQuery = (startAfter : string | null, limit : number, contractAddress: string) : Query => { + + const variables : Record = { + contractAddress, + limit + } + + const query = ` + query ($contractAddress: String!, $limit: Int!){ + wasm { + contractQuery( + contractAddress: $contractAddress, + query: { + pairs: { + limit: $limit + start_after: ${startAfter} + } + } + ) + } + } + ` + + return { query : query, variables : variables } + } + + + private findLatestPair = (pairs : Pair[]) : string | null => { + if (pairs.length === 0) { + return null + } + const latestAssetInfos = this.pairs[this.pairs.length - 1].asset_infos + + let startAfter = `[ + ${this.produceStartAfterAsset(latestAssetInfos[0])}, + ${this.produceStartAfterAsset(latestAssetInfos[1])} + ]` + + return startAfter + } + + private produceStartAfterAsset = (asset : AssetInfo | AssetInfoNative) => { + if ("token" in asset) { + return `{ + token: { + contract_addr: "${asset.token.contract_addr}" + } + }` + } else { + return `{ + native_token: { + denom: "${asset.native_token.denom}" + } + }` + } + } + + producePoolAssets = (assets : Asset[]) : PoolAsset[] => { + + return assets.map((asset) => { + if ("token" in asset.info) { + return { + token: { + denom: asset.info.token.contract_addr, + amount: asset.amount + } + } + } else { + return { + token: { + denom: asset.info.native_token.denom, + amount: asset.amount + } + } + } + }) + + + } +} \ No newline at end of file diff --git a/liquidator/src/amm/OsmosisPoolProvider.ts b/liquidator/src/amm/OsmosisPoolProvider.ts new file mode 100644 index 0000000..7e45d72 --- /dev/null +++ b/liquidator/src/amm/OsmosisPoolProvider.ts @@ -0,0 +1,44 @@ +import { camelCaseKeys } from "../helpers"; +import { Pagination, Pool } from "../types/Pool"; +import { PoolDataProviderInterface } from "./PoolDataProviderInterface"; + +export class OsmosisPoolProvider implements PoolDataProviderInterface { + + constructor(private lcdEndpoint: string) {} + + loadPools = async (): Promise => { + let fetchedAllPools = false + let nextKey = '' + let pools: Pool[] = [] + let totalPoolCount = 0 + while (!fetchedAllPools) { + const response = await fetch( + `${this.lcdEndpoint}/osmosis/gamm/v1beta1/pools${nextKey}`, + ) + const responseJson: any = await response.json() + + if (responseJson.pagination === undefined) { + fetchedAllPools = true + return pools + } + + const pagination = camelCaseKeys(responseJson.pagination) as Pagination + + // osmosis lcd query returns total pool count as 0 after page 1 (but returns the correct count on page 1), so we need to only set it once + if (totalPoolCount === 0) { + totalPoolCount = pagination.total + } + + const poolsRaw = responseJson.pools as Pool[] + + poolsRaw.forEach((pool) => pools.push(camelCaseKeys(pool) as Pool)) + + nextKey = `?pagination.key=${pagination.nextKey}` + if (pools.length >= totalPoolCount) { + fetchedAllPools = true + } + } + + return pools + } +} \ No newline at end of file diff --git a/liquidator/src/amm/PoolDataProviderInterface.ts b/liquidator/src/amm/PoolDataProviderInterface.ts new file mode 100644 index 0000000..31bee4f --- /dev/null +++ b/liquidator/src/amm/PoolDataProviderInterface.ts @@ -0,0 +1,7 @@ +import { Pool } from "../types/Pool"; + +export interface PoolDataProviderInterface { + + loadPools(): Promise + +} \ No newline at end of file diff --git a/liquidator/src/amm/types/AstroportTypes.ts b/liquidator/src/amm/types/AstroportTypes.ts new file mode 100644 index 0000000..085c1b4 --- /dev/null +++ b/liquidator/src/amm/types/AstroportTypes.ts @@ -0,0 +1,73 @@ +export interface Token { + contract_addr: string; + } + + export interface NativeToken { + denom: string; + } + + export interface AssetInfoNative { + native_token: NativeToken; + } + + export interface AssetInfo { + token: Token; + } + + export interface PairType { + xyk: {}; + } + + export interface Pair { + asset_infos: AssetInfoNative[] | AssetInfo[]; + contract_addr: string; + liquidity_token: string; + pair_type: PairType; + } + + export interface ContractQueryPairs { + pairs: Pair[]; + } + + export interface Wasm { + contractQuery: ContractQueryPairs | ContractQueryPool; + } + + export interface Data { + wasm: Wasm; + } + + export interface ResponseData { + data: Data; + } + + // + // Pool type definition + // + + export interface PoolResponseData { + data: Map; + } + + export interface Asset { + info: AssetInfo | AssetInfoNative; + amount: string; + } + + export interface ContractQueryPool { + assets: Asset[]; + total_share: string; + } + + // + // GraphQL + // + export interface Query { + query: string; + variables?: Record; + } + \ No newline at end of file diff --git a/liquidator/src/main.ts b/liquidator/src/main.ts index 361ae49..99f5028 100644 --- a/liquidator/src/main.ts +++ b/liquidator/src/main.ts @@ -9,6 +9,9 @@ import { getConfig as getRoverConfig } from './rover/config/osmosis' import { RoverExecutor } from './rover/RoverExecutor' import { getSecretManager } from './secretManager' import { Network } from './types/network' +import { PoolDataProviderInterface } from './amm/PoolDataProviderInterface.js' +import { OsmosisPoolProvider } from './amm/OsmosisPoolProvider.js' +import { AstroportPoolProvider } from './amm/AstroportPoolProvider.js' const REDBANK = 'Redbank' const ROVER = 'Rover' @@ -24,7 +27,7 @@ export const main = async () => { const addressCount = process.env.MAX_LIQUIDATORS || 1 const paths: HdPath[] = [] - while (paths.length < addressCount) { + while (paths.length < Number(addressCount)) { paths.push(makeCosmoshubPath(paths.length)) } @@ -41,12 +44,15 @@ export const main = async () => { // Produce network const networkEnv = process.env.NETWORK || "LOCALNET" const network = networkEnv === "MAINNET" ? Network.MAINNET : networkEnv === "TESTNET" ? Network.TESTNET : Network.LOCALNET + + const poolProvider = getPoolProvider(process.env.CHAIN_NAME!) + switch (executorType) { case REDBANK: - await launchRedbank(client, queryClient, network, liquidatorMasterAddress) + await launchRedbank(client, queryClient, network, liquidatorMasterAddress, poolProvider) return case ROVER: - await launchRover(client, queryClient, network, liquidatorMasterAddress, liquidator) + await launchRover(client, queryClient, network, liquidatorMasterAddress, liquidator, poolProvider) return default: throw new Error( @@ -55,18 +61,32 @@ export const main = async () => { } } +const getPoolProvider = (chainName: string) : PoolDataProviderInterface => { + switch (chainName) { + case "osmosis": + return new OsmosisPoolProvider(process.env.LCD_ENDPOINT!) + case "neutron": + return new AstroportPoolProvider(process.env.ASTROPORT_FACTORY_CONTRACT!, process.env.GRAPHQL_ENDPOINT!) + default: + throw new Error(`Invalid chain name. Chain name must be either osmosis or neutron, recieved ${chainName}`) + } +} + const launchRover = async ( client: SigningStargateClient, wasmClient: CosmWasmClient, network: Network, liquidatorMasterAddress: string, liquidatorWallet: DirectSecp256k1HdWallet, + poolProvider : PoolDataProviderInterface + ) => { await new RoverExecutor( getRoverConfig(liquidatorMasterAddress, network), client, wasmClient, - liquidatorWallet + liquidatorWallet, + poolProvider ).start() } @@ -75,11 +95,14 @@ const launchRedbank = async ( wasmClient: CosmWasmClient, network: Network, liquidatorAddress: string, + poolProvider : PoolDataProviderInterface + ) => { await new RedbankExecutor( getRedbankConfig(liquidatorAddress, network), client, wasmClient, + poolProvider ).start() } diff --git a/liquidator/src/redbank/RedbankExecutor.ts b/liquidator/src/redbank/RedbankExecutor.ts index 399b10e..a03c1b0 100644 --- a/liquidator/src/redbank/RedbankExecutor.ts +++ b/liquidator/src/redbank/RedbankExecutor.ts @@ -17,6 +17,7 @@ import { BaseExecutor, BaseExecutorConfig } from '../BaseExecutor' import { CosmWasmClient, MsgExecuteContractEncodeObject } from '@cosmjs/cosmwasm-stargate' import { getLargestCollateral, getLargestDebt } from '../liquidationGenerator' import { Collateral, DataResponse } from '../query/types.js' +import { PoolDataProviderInterface } from '../amm/PoolDataProviderInterface.js' const { swapExactAmountIn } = osmosis.gamm.v1beta1.MessageComposer.withTypeUrl @@ -43,8 +44,9 @@ export class RedbankExecutor extends BaseExecutor { config: RedbankExecutorConfig, client: SigningStargateClient, queryClient: CosmWasmClient, + poolProvider: PoolDataProviderInterface ) { - super(config, client, queryClient) + super(config, client, queryClient, poolProvider) this.config = config } diff --git a/liquidator/src/rover/RoverExecutor.ts b/liquidator/src/rover/RoverExecutor.ts index 42d4d52..7546cef 100644 --- a/liquidator/src/rover/RoverExecutor.ts +++ b/liquidator/src/rover/RoverExecutor.ts @@ -60,8 +60,9 @@ export class RoverExecutor extends BaseExecutor { client: SigningStargateClient, queryClient: CosmWasmClient, wallet: DirectSecp256k1HdWallet, + poolProvider: PoolDataProviderInterface, ) { - super(config, client, queryClient) + super(config, client, queryClient, poolProvider) this.config = config this.liquidationActionGenerator = new LiquidationActionGenerator(this.ammRouter) this.wallet = wallet @@ -70,7 +71,9 @@ export class RoverExecutor extends BaseExecutor { // Entry to rover executor start = async () => { await this.initiateRedis() + await this.initiateAstroportPoolProvider() await this.refreshData() + // set up accounts const accounts = await this.wallet.getAccounts() diff --git a/liquidator/test/amm/astroportPoolProvider.test.ts b/liquidator/test/amm/astroportPoolProvider.test.ts new file mode 100644 index 0000000..9f8a181 --- /dev/null +++ b/liquidator/test/amm/astroportPoolProvider.test.ts @@ -0,0 +1,60 @@ +import { AstroportPoolProvider } from "../../src/amm/AstroportPoolProvider" + +describe("Astroport Pool Provider Tests", () => { + const astroportFactoryContract = "neutron1jj0scx400pswhpjes589aujlqagxgcztw04srynmhf0f6zplzn2qqmhwj7" + const poolProvider = new AstroportPoolProvider(astroportFactoryContract,"https://testnet-neutron-gql.marsprotocol.io/graphql/") + + test("We can load pairs", async () => { + const pairs = await poolProvider.fetchPairContracts(astroportFactoryContract) + poolProvider.setPairs(pairs) + expect(poolProvider.getPairs().length).toBeGreaterThan(9) + }) + + test("We can load pools", async () => { + const astroportFactoryContract = "neutron1jj0scx400pswhpjes589aujlqagxgcztw04srynmhf0f6zplzn2qqmhwj7" + const poolProvider = new AstroportPoolProvider(astroportFactoryContract,"https://testnet-neutron-gql.marsprotocol.io/graphql/") + const pairs = await poolProvider.fetchPairContracts(astroportFactoryContract) + poolProvider.setPairs(pairs) + + // verify we parsed correctly + expect(pairs.length).toBeGreaterThan(0) + + // Load pools + const pools = await poolProvider.loadPools() + + // verify pool parsing. We do not do verification of pool assets here, + // as that is done the "We Can Parse Tokens" test below + expect(pools.length).toEqual(pairs.length) + pools.forEach((pool, index) => { + expect(pool.address).toEqual(pairs[index].contract_addr) + }) + }) + + test("We can parse tokens", async () => { + const assets = [ + { + "info": { + "native_token": { + "denom": "ibc/C4CFF46FD6DE35CA4CF4CE031E643C8FDC9BA4B99AE598E9B0ED98FE3A2319F9" + } + }, + "amount": "612796" + }, + { + "info": { + "token": { + "contract_addr": "neutron1h6pztc3fn7h22jm60xx90tk7hln7xg8x0nazef58gqv0n4uw9vqq9khy43" + } + }, + "amount": "6263510" + } + ] + + const poolAssets = poolProvider.producePoolAssets(assets) + expect(poolAssets.length).toBe(2) + expect(poolAssets[0].token.denom).toBe("ibc/C4CFF46FD6DE35CA4CF4CE031E643C8FDC9BA4B99AE598E9B0ED98FE3A2319F9") + expect(poolAssets[0].token.amount).toBe("612796") + expect(poolAssets[1].token.denom).toBe("neutron1h6pztc3fn7h22jm60xx90tk7hln7xg8x0nazef58gqv0n4uw9vqq9khy43") + expect(poolAssets[1].token.amount).toBe("6263510") + }) +}) \ No newline at end of file