From 3262817280b08ae94e0a92417f45a7eae78e9a40 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 14 Sep 2022 00:41:33 +0100 Subject: [PATCH 001/224] Switch upstream GraphQL API endpoint --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 2ddfd5f6..f5e394f4 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -44,7 +44,7 @@ TOKEN_EXPONENT = "9" # Standard Cosmosd SDK REST API endpoint for a node on target network REST_API = "https://api.cheqd.net" # GraphQL API endpoint for target network. Must be sourced from a BigDipper instance. -GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" +GRAPHQL_API = "https://gql.cheqd.io/v1/graphql" ############################################################### From 78071a028f1339fc3fccd13a030797318cdbd170 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 14 Sep 2022 16:54:07 +0100 Subject: [PATCH 002/224] ci: Update upstream API endpoints --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index f5e394f4..2ddfd5f6 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -44,7 +44,7 @@ TOKEN_EXPONENT = "9" # Standard Cosmosd SDK REST API endpoint for a node on target network REST_API = "https://api.cheqd.net" # GraphQL API endpoint for target network. Must be sourced from a BigDipper instance. -GRAPHQL_API = "https://gql.cheqd.io/v1/graphql" +GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" ############################################################### From 45ddb06a6bdf243afb073c1d321faa21a82bb5a1 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 16 Sep 2022 07:10:18 -0300 Subject: [PATCH 003/224] Simplified get accounts query so that it doesn't break --- src/api/bigDipperApi.ts | 230 ++++++++++++++++-------------- src/handlers/circulatingSupply.ts | 1 - 2 files changed, 119 insertions(+), 112 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 7b980476..b750a47a 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -3,115 +3,123 @@ import { Account } from "../types/bigDipper"; import { Coin, Delegation, ValidatorAggregateCountResponse, ValidatorDetailResponse } from "../types/node"; export class BigDipperApi { - constructor(public readonly graphql_client: GraphQLClient) { - } - - async get_accounts(addresses: string[]): Promise { - let query = `query Account($addresses: [String], $utc: timestamp) { - account(where: { address: { _in: $addresses } }) { - address - accountBalances: account_balances(limit: 1, order_by: { height: desc }) { - coins - } - delegations { - amount - } - unbonding: unbonding_delegations( - where: { completion_timestamp: { _gt: $utc } } - ) { - amount - } - redelegations(where: { completion_time: { _gt: $utc } }) { - amount - } - delegationRewards: delegation_rewards { - amount - } - } - }` - - let params = { - utc: new Date(), - addresses - } - - let resp = await this.graphql_client.query<{ account: Account[] }>(query, params); - return resp.account; - } - - async get_account(address: string): Promise { - let accounts = await this.get_accounts([address]); - return accounts[0]; - } - - async get_total_supply(): Promise { - let query = `query Supply { - supply(order_by: {height:desc} limit: 1) { - coins - height - } - }`; - - let resp = await this.graphql_client.query<{ supply: { coins: Coin[] }[] }>(query); - return resp.supply[0].coins; - } - - get_delegator_count_for_validator = async (address: string): Promise => { - let query = `query ValidatorDetails($address: String) { - validator(where: {validator_info: {operator_address: {_eq: $address}}}) { - delegations_aggregate { - aggregate { - count - } - } - } - }` - - const params = { - address: address, - } - - const resp = await this.graphql_client.query(query, params); - if (!resp.validator || !resp.validator.length) { - return 0; - } - - return resp.validator[0].delegations_aggregate.aggregate.count; - } - - get_total_delegator_count = async (): Promise => { - const query = `query ValidatorDetails { - validator { - validatorStatuses: validator_statuses(order_by: {height: desc}, limit: 1) { - jailed - } - delegations { - delegatorAddress: delegator_address - } - } - }` - - const resp = await this.graphql_client.query(query); - const set = new Set(); - resp.validator.forEach((obj, i) => { - if (!obj.validatorStatuses[0]?.jailed) { - obj.delegations.forEach(delegation => { - set.add(delegation.delegatorAddress) - }) - } - }) - - return set.size - } - - get_total_staked_coins = async (): Promise => { - let query = `query StakingInfo{ - staking_pool { - bonded_tokens - } - }` - - const resp = await this.graphql_client.query<{ staking_pool: [{ "bonded_tokens": string }] }>(query); - return resp.staking_pool[0].bonded_tokens; - } + constructor(public readonly graphql_client: GraphQLClient) { + } + + async get_accounts(addresses: string[]): Promise { + let query = `query Account($addresses: [String], $utc: timestamp) { + account(where: {address: {_in: $addresses}}) { + address + } + accountBalances: action_account_balance(where: {address: {_in: $addresses}}) { + address + } + delegations: action_delegation(where: {address: {_in: $addresses}}) { + address + } + unbondingBalance: action_unbonding_delegation_total(where: {address: {_in: $addresses}}) { + address + } + redelegations: action_redelegation(where: {address: {_in: $addresses}}, limit: $limit, offset: $offset, count_total: $pagination) { + redelegations + pagination + } + delegationRewards: action_delegation_reward(where: {address: {_in: $addresses}}) { + validator_address + coins + } + } + }` + + let params = { + // utc: new Date(), + addresses + } + + + try { + let resp = await this.graphql_client.query<{ account: Account[] }>(query, params); + return resp.account; + } catch (e) { + console.error(JSON.stringify(e)) + } + + return []; + } + + async get_account(address: string): Promise { + let accounts = await this.get_accounts([address]); + return accounts[0]; + } + + async get_total_supply(): Promise { + let query = `query Supply { + supply(order_by: {height:desc} limit: 1) { + coins + height + } + }`; + + let resp = await this.graphql_client.query<{ supply: { coins: Coin[] }[] }>(query); + return resp.supply[0].coins; + } + + get_delegator_count_for_validator = async (address: string): Promise => { + let query = `query ValidatorDetails($address: String) { + validator(where: {validator_info: {operator_address: {_eq: $address}}}) { + delegations_aggregate { + aggregate { + count + } + } + } + }` + + const params = { + address: address, + } + + const resp = await this.graphql_client.query(query, params); + if (!resp.validator || !resp.validator.length) { + return 0; + } + + return resp.validator[0].delegations_aggregate.aggregate.count; + } + + get_total_delegator_count = async (): Promise => { + const query = `query ValidatorDetails { + validator { + validatorStatuses: validator_statuses(order_by: {height: desc}, limit: 1) { + jailed + } + delegations { + delegatorAddress: delegator_address + } + } + }` + + const resp = await this.graphql_client.query(query); + const set = new Set(); + resp.validator.forEach((obj, i) => { + if (!obj.validatorStatuses[0]?.jailed) { + obj.delegations.forEach(delegation => { + set.add(delegation.delegatorAddress) + }) + } + }) + + return set.size + } + + get_total_staked_coins = async (): Promise => { + let query = `query StakingInfo{ + staking_pool { + bonded_tokens + } + }` + + const resp = await this.graphql_client.query<{ staking_pool: [{ "bonded_tokens": string }] }>(query); + return resp.staking_pool[0].bonded_tokens; + } } diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index e14fd05e..c98a7bb4 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -12,7 +12,6 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P let node_api = new NodeApi(REST_API); let filtered_accounts = filter_marked_as_account_types(circulating_supply_watchlist); - let non_circulating_accounts = await bd_api.get_accounts(filtered_accounts.other); let non_circulating_accounts_delayed = await Promise.all(filtered_accounts?.delayed?.map(address => node_api.bank_get_account_balances(address))); From b627416363f1e8885c754d7db71a545a06a697c1 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Mon, 19 Sep 2022 15:15:31 -0300 Subject: [PATCH 004/224] Fixed bad query --- src/api/bigDipperApi.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index b750a47a..f8fc7a8c 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -10,8 +10,7 @@ export class BigDipperApi { let query = `query Account($addresses: [String], $utc: timestamp) { account(where: {address: {_in: $addresses}}) { address - } - accountBalances: action_account_balance(where: {address: {_in: $addresses}}) { + accountBalances: action_account_balance(where: {address: {_in: $addresses}}) { address } delegations: action_delegation(where: {address: {_in: $addresses}}) { @@ -28,17 +27,18 @@ export class BigDipperApi { validator_address coins } - } - }` + } + }` let params = { - // utc: new Date(), + utc: new Date(), addresses } try { let resp = await this.graphql_client.query<{ account: Account[] }>(query, params); + console.log({resp}) return resp.account; } catch (e) { console.error(JSON.stringify(e)) @@ -49,9 +49,14 @@ export class BigDipperApi { async get_account(address: string): Promise { let accounts = await this.get_accounts([address]); - return accounts[0]; + if (accounts.length > 0 ){ + return accounts[0]; + } + + return {} as Account } + async get_total_supply(): Promise { let query = `query Supply { supply(order_by: {height:desc} limit: 1) { From 27f81af83bce4e1be7cb2dcfd9869670fe9498ff Mon Sep 17 00:00:00 2001 From: drgomesp Date: Wed, 21 Sep 2022 07:11:35 -0300 Subject: [PATCH 005/224] Improved error handling for graphql client --- src/helpers/graphql.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index 3aaf6660..e68dddb2 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -8,17 +8,22 @@ export class GraphQLClient { variables } - let resp = await fetch(this.base_url, { - method: "POST", - body: JSON.stringify(req) - }) + try { + let resp = await fetch(this.base_url, { + method: "POST", + body: JSON.stringify(req) + }) - let resp_json = await resp.json() as { data: T, errors: any } + let resp_json = await resp.json() as { data: T, errors: any } - if (resp_json.errors != null) { - throw new Error(resp_json.errors); - } + if (resp_json.errors != null) { + console.error(JSON.stringify(resp_json.errors)); + } - return resp_json.data; + return resp_json.data; + } catch (e) { + console.error(JSON.stringify(e)) + return {} as T; + } } } From 158e10be33ca5b5bc47195f7fd3f1eb7c63be164 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Thu, 22 Sep 2022 07:11:33 -0300 Subject: [PATCH 006/224] wip --- src/api/bigDipperApi.ts | 63 +++++++++++-------------------- src/handlers/circulatingSupply.ts | 16 ++++++-- src/helpers/graphql.ts | 11 +++--- src/helpers/node.ts | 18 ++++++--- src/types/bigDipper.ts | 10 +++-- 5 files changed, 58 insertions(+), 60 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index f8fc7a8c..c88293fe 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,59 +1,37 @@ import { GraphQLClient } from "../helpers/graphql"; -import { Account } from "../types/bigDipper"; -import { Coin, Delegation, ValidatorAggregateCountResponse, ValidatorDetailResponse } from "../types/node"; +import { Coin, ValidatorAggregateCountResponse, ValidatorDetailResponse } from "../types/node"; +import { Record } from "../types/bigDipper"; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) { } - async get_accounts(addresses: string[]): Promise { - let query = `query Account($addresses: [String], $utc: timestamp) { - account(where: {address: {_in: $addresses}}) { - address - accountBalances: action_account_balance(where: {address: {_in: $addresses}}) { - address - } - delegations: action_delegation(where: {address: {_in: $addresses}}) { - address - } - unbondingBalance: action_unbonding_delegation_total(where: {address: {_in: $addresses}}) { - address - } - redelegations: action_redelegation(where: {address: {_in: $addresses}}, limit: $limit, offset: $offset, count_total: $pagination) { - redelegations - pagination - } - delegationRewards: action_delegation_reward(where: {address: {_in: $addresses}}) { - validator_address - coins - } - } - }` + async get_accounts(addresses: string[]): Promise { + let query = `query Account($addresses: [String!]) { + account(where: {address: {_in: $addresses}}) { + address + } + account_balance(where: {address: {_in: $addresses}}) { + coins + } +}` let params = { - utc: new Date(), - addresses + addresses: addresses } + let resp = await this.graphql_client.query(query, params); - try { - let resp = await this.graphql_client.query<{ account: Account[] }>(query, params); - console.log({resp}) - return resp.account; - } catch (e) { - console.error(JSON.stringify(e)) - } - - return []; + return resp as Record[] } - async get_account(address: string): Promise { - let accounts = await this.get_accounts([address]); - if (accounts.length > 0 ){ + async get_account(address: string): Promise { + let accounts = await this.get_accounts([ address ]); + if (accounts.length > 0) { return accounts[0]; } - return {} as Account + return {} as Record } @@ -66,7 +44,8 @@ export class BigDipperApi { }`; let resp = await this.graphql_client.query<{ supply: { coins: Coin[] }[] }>(query); - return resp.supply[0].coins; + + return resp.data.supply[0].coins; } get_delegator_count_for_validator = async (address: string): Promise => { @@ -124,7 +103,7 @@ export class BigDipperApi { } }` - const resp = await this.graphql_client.query<{ staking_pool: [{ "bonded_tokens": string }] }>(query); + const resp = await this.graphql_client.query<{ staking_pool: [ { "bonded_tokens": string } ] }>(query); return resp.staking_pool[0].bonded_tokens; } } diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index c98a7bb4..c5500eed 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -16,11 +16,21 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P let non_circulating_accounts_delayed = await Promise.all(filtered_accounts?.delayed?.map(address => node_api.bank_get_account_balances(address))); - // Calculate total balance of watchlist accounts - let non_circulating_supply_ncheq = non_circulating_accounts.map(total_balance_ncheq).reduce((a, b) => a + b, 0); + const { account, account_balance, delegations, unbonding, delegationRewards } = non_circulating_accounts.data; + + let non_circulating_supply_ncheq = 0; + for (let i = 0; i < account.length; i++) { + non_circulating_supply_ncheq += total_balance_ncheq({ + account: account[i], + account_balance: account_balance[i], + delegations: delegations ? delegations[i] : null, + unbonding: unbonding ? unbonding[i] : null, + delegationRewards: delegationRewards ? delegationRewards[i] : null, + }) + } let non_circulating_supply_delayed_ncheq = non_circulating_accounts_delayed.map(account => delayed_balance_ncheq(account)).reduce((a, b) => a + b, 0); - console.log(`Non-circulating supply: ${non_circulating_supply_ncheq + non_circulating_supply_delayed_ncheq}`); + console.log(`Non-circulating supply: ${non_circulating_supply_ncheq + non_circulating_supply_delayed_ncheq}`); // Get total supply let total_supply = await bd_api.get_total_supply(); let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index e68dddb2..9b87cf51 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -5,7 +5,7 @@ export class GraphQLClient { async query(query: string, variables: Object = {}): Promise { let req = { query, - variables + variables, } try { @@ -14,13 +14,14 @@ export class GraphQLClient { body: JSON.stringify(req) }) - let resp_json = await resp.json() as { data: T, errors: any } + let json = await resp.json() - if (resp_json.errors != null) { - console.error(JSON.stringify(resp_json.errors)); + if (json.errors) { + console.error(json.errors) + return {} as T; } - return resp_json.data; + return json as T; } catch (e) { console.error(JSON.stringify(e)) return {} as T; diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 067a3462..7acb083f 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,20 +1,26 @@ -import { Account } from "../types/bigDipper"; import { Coin } from "../types/node"; -export function total_balance_ncheq(account: Account): number { - let balance = Number(account?.accountBalances[0]?.coins.find(c => c.denom === "ncheq")?.amount || '0'); +export function total_balance_ncheq(record: { + account: { address: string } + account_balance: { coins: Coin[] } + delegations: { amount: Coin }[] + unbonding: { amount: Coin }[] + delegationRewards: { amount: Coin[] }[] +}): number { + console.log({ record }) + let balance = Number(record.account_balance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); - let delegations = account?.delegations?.map(d => d.amount) + let delegations = record?.delegations?.map(d => d.amount) .filter(a => a.denom === "ncheq") .map(a => Number(a.amount)) .reduce((a, b) => a + b, 0) ?? 0; - let unbonding = account?.unbonding?.map(d => d.amount) + let unbonding = record?.unbonding?.map(d => d.amount) .filter(a => a.denom === "ncheq") .map(a => Number(a.amount)) .reduce((a, b) => a + b, 0) ?? 0; - let rewards = account?.delegationRewards?.map(d => d.amount) + let rewards = record?.delegationRewards?.map(d => d.amount) .flat() .filter(a => a.denom === "ncheq") .map(a => Number(a.amount)) diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index e09a863a..183a1f51 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -1,13 +1,15 @@ import { Coin } from "./node"; -export class Account { - public accountBalances: { coins: Coin[] }[]; +export class Record { + public account: { address: String }[]; + public account_balance: { coins: Coin[] }[]; public delegations: { amount: Coin }[]; public unbonding: { amount: Coin }[]; public delegationRewards: { amount: Coin[] }[]; - constructor(accountBalances: { coins: Coin[] }[], delegations: { amount: Coin }[], unbonding: { amount: Coin }[], delegationRewards: { amount: Coin[] }[]) { - this.accountBalances = accountBalances; + constructor(account: { address: String }[], account_balance: { coins: Coin[] }[], delegations: { amount: Coin }[], unbonding: { amount: Coin }[], delegationRewards: { amount: Coin[] }[]) { + this.account = account; + this.account_balance = account_balance; this.delegations = delegations; this.unbonding = unbonding; this.delegationRewards = delegationRewards; From 59e1bf590fe63027941f9fd61e1798dd508dc1fb Mon Sep 17 00:00:00 2001 From: drgomesp Date: Thu, 22 Sep 2022 13:51:58 -0300 Subject: [PATCH 007/224] Fixed circulating supply endpoint (still missing rewards and unbonding) --- src/api/bigDipperApi.ts | 29 +++++++++++++------- src/handlers/circulatingSupply.ts | 12 ++++----- src/handlers/totalBalance.ts | 15 +++++++---- src/helpers/node.ts | 44 ++++++++++++++----------------- src/types/bigDipper.ts | 7 ++--- 5 files changed, 59 insertions(+), 48 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index c88293fe..e36333fc 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -8,13 +8,13 @@ export class BigDipperApi { async get_accounts(addresses: string[]): Promise { let query = `query Account($addresses: [String!]) { - account(where: {address: {_in: $addresses}}) { - address - } - account_balance(where: {address: {_in: $addresses}}) { - coins - } -}` + account(where: {address: {_in: $addresses}}) { + address + } + account_balance(where: {address: {_in: $addresses}}) { + coins + } + }` let params = { addresses: addresses @@ -22,13 +22,22 @@ export class BigDipperApi { let resp = await this.graphql_client.query(query, params); + console.log({ resp }) + return resp as Record[] } async get_account(address: string): Promise { - let accounts = await this.get_accounts([ address ]); - if (accounts.length > 0) { - return accounts[0]; + const record = await this.get_accounts([ address ]); + + const { account, account_balance, delegations, unbonding, delegationRewards } = record.data; + if (account.length > 0) { + + return { + account: account[0], + account_balance: account_balance[0], + delegationRewards: [], delegations: [], unbonding: [] + } as Record; } return {} as Record diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index c5500eed..b6a66c5f 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -5,6 +5,7 @@ import { Request } from "itty-router"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; import { delayed_balance_ncheq, total_balance_ncheq } from "../helpers/node"; import { filter_marked_as_account_types } from '../helpers/validate'; +import { Record } from "../types/bigDipper"; async function get_circulating_supply(circulating_supply_watchlist: string[]): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); @@ -13,20 +14,19 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P let filtered_accounts = filter_marked_as_account_types(circulating_supply_watchlist); let non_circulating_accounts = await bd_api.get_accounts(filtered_accounts.other); - let non_circulating_accounts_delayed = await Promise.all(filtered_accounts?.delayed?.map(address => node_api.bank_get_account_balances(address))); - const { account, account_balance, delegations, unbonding, delegationRewards } = non_circulating_accounts.data; + const { account, account_balance, } = non_circulating_accounts.data; let non_circulating_supply_ncheq = 0; for (let i = 0; i < account.length; i++) { non_circulating_supply_ncheq += total_balance_ncheq({ account: account[i], account_balance: account_balance[i], - delegations: delegations ? delegations[i] : null, - unbonding: unbonding ? unbonding[i] : null, - delegationRewards: delegationRewards ? delegationRewards[i] : null, - }) + // delegations: delegations ? delegations[i] : null, + // unbonding: unbonding ? unbonding[i] : null, + // delegationRewards: delegationRewards ? delegationRewards[i] : null, + } as Record) } let non_circulating_supply_delayed_ncheq = non_circulating_accounts_delayed.map(account => delayed_balance_ncheq(account)).reduce((a, b) => a + b, 0); diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index c1bc6afa..2cc699a8 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,5 +1,5 @@ import { Request } from "itty-router"; -import { is_delayed_vesting_account_type, validate_cheqd_address} from "../helpers/validate"; +import { is_delayed_vesting_account_type, validate_cheqd_address } from "../helpers/validate"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; import { BigDipperApi } from "../api/bigDipperApi"; import { GraphQLClient } from "../helpers/graphql"; @@ -16,7 +16,7 @@ export async function handler(request: Request): Promise { let node_api = new NodeApi(REST_API); let auth_account = await node_api.auth_get_account(address); - if(is_delayed_vesting_account_type(auth_account?.["@type"])) { + if (is_delayed_vesting_account_type(auth_account?.["@type"])) { let balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); let rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); let delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); @@ -27,8 +27,13 @@ export async function handler(request: Request): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - let account = await bd_api.get_account(address); - let balance = total_balance_ncheq(account); + const record = await bd_api.get_account(address); - return new Response(ncheq_to_cheq_fixed(balance)) + if (record.account) { + let balance = total_balance_ncheq(record); + + return new Response(ncheq_to_cheq_fixed(balance)) + } + + return new Response("no accounts found", { status: 500 }) } diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 7acb083f..5124854e 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,32 +1,28 @@ import { Coin } from "../types/node"; +import { Record } from "../types/bigDipper"; -export function total_balance_ncheq(record: { - account: { address: string } - account_balance: { coins: Coin[] } - delegations: { amount: Coin }[] - unbonding: { amount: Coin }[] - delegationRewards: { amount: Coin[] }[] -}): number { - console.log({ record }) +export function total_balance_ncheq(record: Record): number { let balance = Number(record.account_balance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); - let delegations = record?.delegations?.map(d => d.amount) - .filter(a => a.denom === "ncheq") - .map(a => Number(a.amount)) - .reduce((a, b) => a + b, 0) ?? 0; + console.log(record) + // let delegations = record?.delegations?.map(d => d.amount) + // .filter(a => a.denom === "ncheq") + // .map(a => Number(a.amount)) + // .reduce((a, b) => a + b, 0) ?? 0; + // + // let unbonding = record?.unbonding?.map(d => d.amount) + // .filter(a => a.denom === "ncheq") + // .map(a => Number(a.amount)) + // .reduce((a, b) => a + b, 0) ?? 0; + // + // let rewards = record?.delegationRewards?.map(d => d.amount) + // .flat() + // .filter(a => a.denom === "ncheq") + // .map(a => Number(a.amount)) + // .reduce((a, b) => a + b, 0) ?? 0; - let unbonding = record?.unbonding?.map(d => d.amount) - .filter(a => a.denom === "ncheq") - .map(a => Number(a.amount)) - .reduce((a, b) => a + b, 0) ?? 0; - - let rewards = record?.delegationRewards?.map(d => d.amount) - .flat() - .filter(a => a.denom === "ncheq") - .map(a => Number(a.amount)) - .reduce((a, b) => a + b, 0) ?? 0; - - return balance + delegations + unbonding + rewards; + return balance; + // return balance + delegations + unbonding + rewards; } export function delayed_balance_ncheq(balance: Coin[]): number { diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index 183a1f51..090bcd3c 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -1,13 +1,13 @@ import { Coin } from "./node"; export class Record { - public account: { address: String }[]; - public account_balance: { coins: Coin[] }[]; + public account: { address: String }; + public account_balance: { coins: Coin[] }; public delegations: { amount: Coin }[]; public unbonding: { amount: Coin }[]; public delegationRewards: { amount: Coin[] }[]; - constructor(account: { address: String }[], account_balance: { coins: Coin[] }[], delegations: { amount: Coin }[], unbonding: { amount: Coin }[], delegationRewards: { amount: Coin[] }[]) { + constructor(account: { address: String }, account_balance: { coins: Coin[] }, delegations: { amount: Coin }[], unbonding: { amount: Coin }[], delegationRewards: { amount: Coin[] }[]) { this.account = account; this.account_balance = account_balance; this.delegations = delegations; @@ -15,3 +15,4 @@ export class Record { this.delegationRewards = delegationRewards; } } + From b0b8081ac51184a3adac2dda661d0234c2553ac4 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 14 Sep 2022 00:41:33 +0100 Subject: [PATCH 008/224] Switch upstream GraphQL API endpoint --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 2ddfd5f6..f5e394f4 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -44,7 +44,7 @@ TOKEN_EXPONENT = "9" # Standard Cosmosd SDK REST API endpoint for a node on target network REST_API = "https://api.cheqd.net" # GraphQL API endpoint for target network. Must be sourced from a BigDipper instance. -GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" +GRAPHQL_API = "https://gql.cheqd.io/v1/graphql" ############################################################### From 5dbe3e2c5838b4149af8bb3e565136b92227c892 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 14 Sep 2022 16:54:07 +0100 Subject: [PATCH 009/224] ci: Update upstream API endpoints --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index f5e394f4..2ddfd5f6 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -44,7 +44,7 @@ TOKEN_EXPONENT = "9" # Standard Cosmosd SDK REST API endpoint for a node on target network REST_API = "https://api.cheqd.net" # GraphQL API endpoint for target network. Must be sourced from a BigDipper instance. -GRAPHQL_API = "https://gql.cheqd.io/v1/graphql" +GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" ############################################################### From 078d6234c11802b8cb7cd46d28053aa13666571c Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Mon, 19 Sep 2022 15:15:31 -0300 Subject: [PATCH 010/224] Fixed bad query --- src/api/bigDipperApi.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index e36333fc..49d8495e 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -44,6 +44,7 @@ export class BigDipperApi { } + async get_total_supply(): Promise { let query = `query Supply { supply(order_by: {height:desc} limit: 1) { From 68327561402fff23d47ebce67b472e07d82de47b Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 23 Sep 2022 06:22:37 -0300 Subject: [PATCH 011/224] Created new endpoint to update single account balance on KV for later usage --- src/api/bigDipperApi.ts | 2 +- src/handlers/balanceUpdater.ts | 49 ++++++++++++++++++++++++++++++++++ src/index.ts | 8 +++--- wrangler.toml | 6 ++--- 4 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 src/handlers/balanceUpdater.ts diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 49d8495e..9b0f0a4e 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -31,8 +31,8 @@ export class BigDipperApi { const record = await this.get_accounts([ address ]); const { account, account_balance, delegations, unbonding, delegationRewards } = record.data; - if (account.length > 0) { + if (account.length > 0) { return { account: account[0], account_balance: account_balance[0], diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts new file mode 100644 index 00000000..2bca2a82 --- /dev/null +++ b/src/handlers/balanceUpdater.ts @@ -0,0 +1,49 @@ +import { NodeApi } from "../api/nodeApi"; +import { Request } from "itty-router"; +import { validate_cheqd_address } from "../helpers/validate"; +import { ncheq_to_cheqd } from "../helpers/currency"; + +export async function handler(request: Request): Promise { + let node_api = new NodeApi(REST_API); + const address = request.params?.['address']; + + if (!address || !validate_cheqd_address(address)) { + throw new Error("No address specified or wrong address format."); + } + + const account = await node_api.auth_get_account(address) + + try { + const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(address) + + if (cachedAccount) { + console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) + } + } catch (e) { + console.error(e) + } + + const balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); + const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); + const delegated = Number(account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); + + const res = { + balance: ncheq_to_cheqd(balance), + rewards: ncheq_to_cheqd(rewards), + delegated: ncheq_to_cheqd(delegated), + }; + + let totalBalance = res.balance + res.rewards + res.delegated + + console.log(`account "${address}" total balance: ${totalBalance}`) + + try { + await CIRCULATING_SUPPLY_WATCHLIST.put(address, JSON.stringify(res)) + console.log(`account "${address}" balance updated.`) + } catch (e) { + console.error(e) + } + + + return new Response(JSON.stringify(res)); +} diff --git a/src/index.ts b/src/index.ts index 6f99d593..1bcaa48a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import { Router, Request, IHTTPMethods } from 'itty-router' +import { IHTTPMethods, Request, Router } from 'itty-router' import { handler as totalSupplyHandler } from "./handlers/totalSupply"; import { handler as totalBalanceHandler } from "./handlers/totalBalance"; import { handler as circulatingSupplyHandler } from "./handlers/circulatingSupply"; @@ -27,10 +27,10 @@ function registerRoutes(router: Router) { router.get('/supply/staked', totalStakedCoinsHandler); router.get('/supply/total', totalSupplyHandler); - // 404 for all other requests - router.all('*', () => new Response('Not Found.', { status: 404 })) + // 404 for all other requests + router.all('*', () => new Response('Not Found.', { status: 404 })) } function handleError(error: Error): Response { - return new Response(error.message || 'Server Error', { status: 500 }) + return new Response(error.message || 'Server Error', { status: 500 }) } diff --git a/wrangler.toml b/wrangler.toml index 2ddfd5f6..1e26d0b5 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -9,7 +9,7 @@ name = "data-api" main = "src/index.ts" # Date in yyyy-mm-dd to determine which version of Workers runtime to use -# Details: https://developers.cloudflare.com/workers/platform/compatibility-dates/ +# Details: https://developers.cloudflare.com/workers/platform/compatibility-dates/ compatibility_date = "2022-05-24" # Usage model for the Worker @@ -98,13 +98,13 @@ vars = { TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = # Details: https://developers.cloudflare.com/workers/wrangler/custom-builds/ # @default {} # [build] -# # Custom build command. On Linux and macOS, the command is executed in +# # Custom build command. On Linux and macOS, the command is executed in # # the `sh` shell and the `cmd` shell for Windows. # # The `&&` and `||` shell operators may be used. # command = "" # # The directory in which the command is executed. # cwd = "./src" -# # The directory to watch for changes while using `wrangler dev`, +# # The directory to watch for changes while using `wrangler dev`, # # defaults to the current working directory # watch_dir = "" From b1915757d92d83d6e652a62c608e6bd7cd5e852a Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 23 Sep 2022 06:33:00 -0300 Subject: [PATCH 012/224] Added total balance as field on account balance cache --- src/handlers/balanceUpdater.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index 2bca2a82..d855330a 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -16,7 +16,7 @@ export async function handler(request: Request): Promise { try { const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(address) - if (cachedAccount) { + if (typeof cachedAccount === "object") { console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) } } catch (e) { @@ -27,15 +27,14 @@ export async function handler(request: Request): Promise { const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); const delegated = Number(account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); - const res = { + let res = { balance: ncheq_to_cheqd(balance), rewards: ncheq_to_cheqd(rewards), delegated: ncheq_to_cheqd(delegated), + total_balance: ncheq_to_cheqd(balance + rewards + delegated) }; - let totalBalance = res.balance + res.rewards + res.delegated - - console.log(`account "${address}" total balance: ${totalBalance}`) + console.log(`account "${address}" total balance: ${res.total_balance}`) try { await CIRCULATING_SUPPLY_WATCHLIST.put(address, JSON.stringify(res)) From 88da7d3f26e43f1b866a6ea35f87650766d5ae98 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 23 Sep 2022 12:44:57 +0200 Subject: [PATCH 013/224] Update bigDipperApi.ts --- src/api/bigDipperApi.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 9b0f0a4e..2a534557 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -16,13 +16,21 @@ export class BigDipperApi { } }` + let query2 = `query AccountDelegations($addresses: [String!]) { + delegations: action_delegation(where: {address: {_in: $addresses}}) { + coins + } + }` + let params = { addresses: addresses } let resp = await this.graphql_client.query(query, params); + let resp2 = await this.graphql_client.query(query2, params); console.log({ resp }) + console.log({ resp2 }) return resp as Record[] } From a50df0dcec56cf97814c81604c800de530298b57 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 23 Sep 2022 13:31:12 +0200 Subject: [PATCH 014/224] Revert "Update bigDipperApi.ts" This reverts commit 50b08ee5f68c633c7dd87683e627c674c61e61fa. --- src/api/bigDipperApi.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 2a534557..9b0f0a4e 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -16,21 +16,13 @@ export class BigDipperApi { } }` - let query2 = `query AccountDelegations($addresses: [String!]) { - delegations: action_delegation(where: {address: {_in: $addresses}}) { - coins - } - }` - let params = { addresses: addresses } let resp = await this.graphql_client.query(query, params); - let resp2 = await this.graphql_client.query(query2, params); console.log({ resp }) - console.log({ resp2 }) return resp as Record[] } From ed2491fbfe5cced951fca2c5798f73dc646b2df1 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Mon, 26 Sep 2022 07:36:00 -0300 Subject: [PATCH 015/224] Refactoring for balance fetching from different sources --- src/api/bigDipperApi.ts | 16 +++------ src/handlers/balanceUpdater.ts | 39 ++------------------- src/handlers/circulatingSupply.ts | 57 +++++++++++++++++-------------- src/handlers/cron.ts | 29 ++++++++++++++++ src/helpers/balance.ts | 38 +++++++++++++++++++++ src/helpers/node.ts | 5 ++- src/index.ts | 9 +++++ src/types/bigDipper.ts | 15 ++++++++ wrangler.toml | 33 +++++++++--------- 9 files changed, 152 insertions(+), 89 deletions(-) create mode 100644 src/handlers/cron.ts create mode 100644 src/helpers/balance.ts diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 9b0f0a4e..290ee89c 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,30 +1,25 @@ import { GraphQLClient } from "../helpers/graphql"; import { Coin, ValidatorAggregateCountResponse, ValidatorDetailResponse } from "../types/node"; -import { Record } from "../types/bigDipper"; +import { Record, Records } from "../types/bigDipper"; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) { } - async get_accounts(addresses: string[]): Promise { + async get_accounts(addresses: string[]): Promise { let query = `query Account($addresses: [String!]) { account(where: {address: {_in: $addresses}}) { address } - account_balance(where: {address: {_in: $addresses}}) { - coins - } }` let params = { addresses: addresses } - let resp = await this.graphql_client.query(query, params); - - console.log({ resp }) + let resp = await this.graphql_client.query(query, params); - return resp as Record[] + return resp as Records[] } async get_account(address: string): Promise { @@ -35,8 +30,6 @@ export class BigDipperApi { if (account.length > 0) { return { account: account[0], - account_balance: account_balance[0], - delegationRewards: [], delegations: [], unbonding: [] } as Record; } @@ -55,6 +48,7 @@ export class BigDipperApi { let resp = await this.graphql_client.query<{ supply: { coins: Coin[] }[] }>(query); + console.log(resp) return resp.data.supply[0].coins; } diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index d855330a..eaff1f5e 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -1,48 +1,15 @@ import { NodeApi } from "../api/nodeApi"; import { Request } from "itty-router"; import { validate_cheqd_address } from "../helpers/validate"; -import { ncheq_to_cheqd } from "../helpers/currency"; +import { updateBalance } from "../helpers/balance"; export async function handler(request: Request): Promise { let node_api = new NodeApi(REST_API); const address = request.params?.['address']; if (!address || !validate_cheqd_address(address)) { - throw new Error("No address specified or wrong address format."); + throw new Error("missing/invalid account address") } - const account = await node_api.auth_get_account(address) - - try { - const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(address) - - if (typeof cachedAccount === "object") { - console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) - } - } catch (e) { - console.error(e) - } - - const balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); - const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); - const delegated = Number(account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); - - let res = { - balance: ncheq_to_cheqd(balance), - rewards: ncheq_to_cheqd(rewards), - delegated: ncheq_to_cheqd(delegated), - total_balance: ncheq_to_cheqd(balance + rewards + delegated) - }; - - console.log(`account "${address}" total balance: ${res.total_balance}`) - - try { - await CIRCULATING_SUPPLY_WATCHLIST.put(address, JSON.stringify(res)) - console.log(`account "${address}" balance updated.`) - } catch (e) { - console.error(e) - } - - - return new Response(JSON.stringify(res)); + return updateBalance(node_api, address); } diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index b6a66c5f..5c50d7e2 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,43 +1,50 @@ import { GraphQLClient } from "../helpers/graphql"; import { BigDipperApi } from "../api/bigDipperApi"; -import { NodeApi } from '../api/nodeApi'; import { Request } from "itty-router"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; -import { delayed_balance_ncheq, total_balance_ncheq } from "../helpers/node"; +import { cheqd_to_ncheq, ncheq_to_cheq_fixed } from "../helpers/currency"; import { filter_marked_as_account_types } from '../helpers/validate'; -import { Record } from "../types/bigDipper"; async function get_circulating_supply(circulating_supply_watchlist: string[]): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - let node_api = new NodeApi(REST_API); let filtered_accounts = filter_marked_as_account_types(circulating_supply_watchlist); let non_circulating_accounts = await bd_api.get_accounts(filtered_accounts.other); - let non_circulating_accounts_delayed = await Promise.all(filtered_accounts?.delayed?.map(address => node_api.bank_get_account_balances(address))); - - const { account, account_balance, } = non_circulating_accounts.data; - - let non_circulating_supply_ncheq = 0; - for (let i = 0; i < account.length; i++) { - non_circulating_supply_ncheq += total_balance_ncheq({ - account: account[i], - account_balance: account_balance[i], - // delegations: delegations ? delegations[i] : null, - // unbonding: unbonding ? unbonding[i] : null, - // delegationRewards: delegationRewards ? delegationRewards[i] : null, - } as Record) - } - let non_circulating_supply_delayed_ncheq = non_circulating_accounts_delayed.map(account => delayed_balance_ncheq(account)).reduce((a, b) => a + b, 0); - console.log(`Non-circulating supply: ${non_circulating_supply_ncheq + non_circulating_supply_delayed_ncheq}`); - // Get total supply let total_supply = await bd_api.get_total_supply(); let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); - console.log(`Total supply: ${total_supply_ncheq}`); - // Calculate circulating supply - return total_supply_ncheq - non_circulating_supply_ncheq - non_circulating_supply_delayed_ncheq; + if (typeof non_circulating_accounts === 'object') { + try { + + const cachedBalances = await CIRCULATING_SUPPLY_WATCHLIST.list({ + prefix: "grp_1." + }) + let non_circulating_supply_ncheq = 0; + for (const account of cachedBalances.keys) { + const k = account.name.split('.') + console.log(`getting from KV: ${k[1]}`) + + let cached = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${k[1]}`); + if (cached) { + const data = JSON.parse(cached) + non_circulating_supply_ncheq += cheqd_to_ncheq(data.total_balance); + } + + } + console.log(`Non-circulating supply: ${non_circulating_supply_ncheq}`); + // Get total supply + let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); + console.log(`Total supply: ${total_supply_ncheq}`); + + // Calculate circulating supply + return total_supply_ncheq - non_circulating_supply_ncheq; + } catch (e) { + console.error(e) + } + + return total_supply_ncheq + } } export async function handler(request: Request): Promise { diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts new file mode 100644 index 00000000..aa929ef5 --- /dev/null +++ b/src/handlers/cron.ts @@ -0,0 +1,29 @@ +import { NodeApi } from "../api/nodeApi"; +import { updateBalance } from "../helpers/balance"; + +export async function updateAllBalances(event: Event) { + let node_api = new NodeApi(REST_API); + let balances: { account: String, balances: {}[] } [] = []; + + try { + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); + + console.log(`found ${cached.keys.length} cached accounts`) + for (const account of cached.keys) { + let addr: string; + if (account.name.startsWith("grp_1.")) { + const parts = account.name.split('.') + addr = parts[1] + } else { + addr = account.name + } + const res = await updateBalance(node_api, addr) + + balances.push({ account: addr, balances: await res.json() }) + } + + return new Response(JSON.stringify(balances)); + } catch (e) { + console.error(e) + } +} diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts new file mode 100644 index 00000000..0d9b0c5e --- /dev/null +++ b/src/helpers/balance.ts @@ -0,0 +1,38 @@ +import { NodeApi } from "../api/nodeApi"; +import { ncheq_to_cheqd } from "./currency"; + +export async function updateBalance(node_api: NodeApi, address: string): Promise { + const account = await node_api.auth_get_account(address) + + try { + const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${address}`) + + if (typeof cachedAccount === "object") { + console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) + } + + const balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); + const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); + const delegated = Number(account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); + + let res = { + balance: ncheq_to_cheqd(balance), + rewards: ncheq_to_cheqd(rewards), + delegated: ncheq_to_cheqd(delegated), + total_balance: ncheq_to_cheqd(balance + rewards + delegated) + }; + + console.log(`account "${address}" total balance: ${res.total_balance}`) + + try { + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_1.${address}`, JSON.stringify(res)) + console.log(`account "${address}" balance updated. (res=${JSON.stringify(res)})`) + + return new Response(JSON.stringify(res)); + } catch (e) { + console.error(e) + } + } catch (e) { + console.error(e) + } +} diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 5124854e..43d19287 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,8 +1,11 @@ import { Coin } from "../types/node"; import { Record } from "../types/bigDipper"; +import { NodeApi } from "../api/nodeApi"; export function total_balance_ncheq(record: Record): number { - let balance = Number(record.account_balance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); + // let balance = Number(record.account_balance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); + let node_api = new NodeApi(REST_API); + // let balance = node_api.bank_get_account_balances(record.account.address.toString()) console.log(record) // let delegations = record?.delegations?.map(d => d.amount) diff --git a/src/index.ts b/src/index.ts index 1bcaa48a..977e9f92 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,6 +8,12 @@ import { handler as vestedBalanceHandler } from "./handlers/vestedBalance"; import { handler as delegatorCountHandler } from './handlers/delegatorCount'; import { handler as totalDelegatorsHandler } from './handlers/totalDelegators'; import { handler as totalStakedCoinsHandler } from "./handlers/totalStakedCoins"; +import { handler as balanceUpdaterHandler } from "./handlers/balanceUpdater"; +import { updateAllBalances } from "./handlers/cron"; + +addEventListener('scheduled', (event: any) => { + event.waitUntil(updateAllBalances(event)); +}) addEventListener('fetch', (event: FetchEvent) => { const router = Router() @@ -26,6 +32,9 @@ function registerRoutes(router: Router) { router.get('/supply/circulating', circulatingSupplyHandler); router.get('/supply/staked', totalStakedCoinsHandler); router.get('/supply/total', totalSupplyHandler); + router.get('/', totalSupplyHandler); + router.get('/_', balanceUpdaterHandler); + router.get('/_/:address', balanceUpdaterHandler); // 404 for all other requests router.all('*', () => new Response('Not Found.', { status: 404 })) diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index 090bcd3c..5d56a805 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -1,5 +1,20 @@ import { Coin } from "./node"; +export class Records { + public accountBalances: { coins: Coin[] }[]; + public delegations: { amount: Coin }[]; + public unbonding: { amount: Coin }[]; + public delegationRewards: { amount: Coin[] }[]; + + constructor(accountBalances: { coins: Coin[] }[], delegations: { amount: Coin }[], unbonding: { amount: Coin }[], delegationRewards: { amount: Coin[] }[]) { + this.accountBalances = accountBalances; + this.delegations = delegations; + this.unbonding = unbonding; + this.delegationRewards = delegationRewards; + } +} + + export class Record { public account: { address: String }; public account_balance: { coins: Coin[] }; diff --git a/wrangler.toml b/wrangler.toml index 1e26d0b5..5fdaec9f 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -14,7 +14,7 @@ compatibility_date = "2022-05-24" # Usage model for the Worker # Details: https://developers.cloudflare.com/workers/platform/limits -usage_model = "bundled" +usage_model = "unbound" ############################################################### @@ -32,9 +32,9 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # Details: https://developers.cloudflare.com/workers/learning/how-kv-works # @default `[]` [[kv_namespaces]] - binding = "CIRCULATING_SUPPLY_WATCHLIST" - id = "ce2e547de27f467c86afa67996b8784a" - preview_id = "a9b6ebe3d1b44671acafb0ed5b7f8919" +binding = "CIRCULATING_SUPPLY_WATCHLIST" +id = "ce2e547de27f467c86afa67996b8784a" +preview_id = "a9b6ebe3d1b44671acafb0ed5b7f8919" # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` @@ -53,15 +53,15 @@ GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" # Configuration options for local development via Wrangler [dev] - # IP address for the local dev server to listen on - # @default `localhost` - ip = "localhost" - # Port for the local dev server to listen on - # @default `8787` - port = 8787 - # Protocol that local wrangler dev server listens to requests on - # @default `http` - local_protocol = "http" +# IP address for the local dev server to listen on +# @default `localhost` +ip = "localhost" +# Port for the local dev server to listen on +# @default `8787` +port = 8787 +# Protocol that local wrangler dev server listens to requests on +# @default `http` +local_protocol = "http" ############################################################### @@ -84,10 +84,11 @@ vars = { TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = # Details: https://developers.cloudflare.com/workers/learning/how-kv-works # @default `[]` [[env.staging.kv_namespaces]] - binding = "CIRCULATING_SUPPLY_WATCHLIST" - id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" - preview_id = "1e1032cbf6854d88b12317da8a792928" +binding = "CIRCULATING_SUPPLY_WATCHLIST" +id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" +preview_id = "1e1032cbf6854d88b12317da8a792928" +crons = ["0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 935a67992514db9fb827f98d5757fac33fe6eb66 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Mon, 26 Sep 2022 15:02:47 -0300 Subject: [PATCH 016/224] Fixed account balance helper --- src/handlers/totalBalance.ts | 30 ++-------------------- src/helpers/balance.ts | 6 +++-- src/helpers/node.ts | 49 ++++++++++++++++++------------------ 3 files changed, 31 insertions(+), 54 deletions(-) diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 2cc699a8..0fa5f1d6 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,10 +1,6 @@ import { Request } from "itty-router"; -import { is_delayed_vesting_account_type, validate_cheqd_address } from "../helpers/validate"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; -import { BigDipperApi } from "../api/bigDipperApi"; -import { GraphQLClient } from "../helpers/graphql"; +import { validate_cheqd_address } from "../helpers/validate"; import { total_balance_ncheq } from "../helpers/node"; -import { NodeApi } from "../api/nodeApi"; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -13,27 +9,5 @@ export async function handler(request: Request): Promise { throw new Error("No address specified or wrong address format."); } - let node_api = new NodeApi(REST_API); - let auth_account = await node_api.auth_get_account(address); - - if (is_delayed_vesting_account_type(auth_account?.["@type"])) { - let balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); - let rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); - let delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); - - return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); - } - - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); - - const record = await bd_api.get_account(address); - - if (record.account) { - let balance = total_balance_ncheq(record); - - return new Response(ncheq_to_cheq_fixed(balance)) - } - - return new Response("no accounts found", { status: 500 }) + return new Response(await total_balance_ncheq(address)) } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 0d9b0c5e..c4cc2336 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,8 +1,10 @@ import { NodeApi } from "../api/nodeApi"; import { ncheq_to_cheqd } from "./currency"; +import { total_balance_ncheq } from "./node"; export async function updateBalance(node_api: NodeApi, address: string): Promise { const account = await node_api.auth_get_account(address) + const auth_account = await node_api.auth_get_account(address); try { const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${address}`) @@ -13,13 +15,13 @@ export async function updateBalance(node_api: NodeApi, address: string): Promise const balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); - const delegated = Number(account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); + const delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); let res = { balance: ncheq_to_cheqd(balance), rewards: ncheq_to_cheqd(rewards), delegated: ncheq_to_cheqd(delegated), - total_balance: ncheq_to_cheqd(balance + rewards + delegated) + total_balance: await total_balance_ncheq(address) }; console.log(`account "${address}" total balance: ${res.total_balance}`) diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 43d19287..644c56de 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,31 +1,32 @@ import { Coin } from "../types/node"; -import { Record } from "../types/bigDipper"; import { NodeApi } from "../api/nodeApi"; +import { is_delayed_vesting_account_type } from "./validate"; +import { ncheq_to_cheq_fixed } from "./currency"; -export function total_balance_ncheq(record: Record): number { +export async function total_balance_ncheq(address: string): Promise { // let balance = Number(record.account_balance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); - let node_api = new NodeApi(REST_API); - // let balance = node_api.bank_get_account_balances(record.account.address.toString()) - - console.log(record) - // let delegations = record?.delegations?.map(d => d.amount) - // .filter(a => a.denom === "ncheq") - // .map(a => Number(a.amount)) - // .reduce((a, b) => a + b, 0) ?? 0; - // - // let unbonding = record?.unbonding?.map(d => d.amount) - // .filter(a => a.denom === "ncheq") - // .map(a => Number(a.amount)) - // .reduce((a, b) => a + b, 0) ?? 0; - // - // let rewards = record?.delegationRewards?.map(d => d.amount) - // .flat() - // .filter(a => a.denom === "ncheq") - // .map(a => Number(a.amount)) - // .reduce((a, b) => a + b, 0) ?? 0; - - return balance; - // return balance + delegations + unbonding + rewards; + const node_api = new NodeApi(REST_API); + const auth_account = await node_api.auth_get_account(address); + + const balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); + const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); + + if (is_delayed_vesting_account_type(auth_account?.["@type"])) { + console.log('is delayed vesting account') + const delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); + + return Number(ncheq_to_cheq_fixed(balance + rewards + delegated)); + } + + const delegated = Number(auth_account?.base_vesting_account?.delegated_free?.find(d => d.denom === "ncheq")?.amount ?? '0'); + + console.log({ + delegated_free: auth_account?.base_vesting_account?.delegated_free, + delegated_vesting: auth_account?.base_vesting_account?.delegated_vesting, + original_vesting: auth_account?.base_vesting_account?.original_vesting + }) + + return Number(ncheq_to_cheq_fixed(balance + delegated + rewards)); } export function delayed_balance_ncheq(balance: Coin[]): number { From 033132b6a0553929db694086d1a1bda928fde5ca Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 05:44:46 -0300 Subject: [PATCH 017/224] Started with new graphql query --- src/api/bigDipperApi.ts | 56 ++++++++++++++++++++---------------- src/handlers/totalBalance.ts | 26 +++++++++++++++-- src/helpers/node.ts | 38 +++++++++++------------- src/types/bigDipper.ts | 56 +++++++++++++++++++----------------- 4 files changed, 101 insertions(+), 75 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 290ee89c..d3baf4bf 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,43 +1,51 @@ import { GraphQLClient } from "../helpers/graphql"; import { Coin, ValidatorAggregateCountResponse, ValidatorDetailResponse } from "../types/node"; -import { Record, Records } from "../types/bigDipper"; +import { Account } from "../types/bigDipper"; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) { } - async get_accounts(addresses: string[]): Promise { - let query = `query Account($addresses: [String!]) { - account(where: {address: {_in: $addresses}}) { - address - } + async get_account(address: string): Promise { + let query = `query Account($address: String!, $where: vesting_account_bool_exp) { + accountBalance: action_account_balance(address: $address) { + coins + } + delegationBalance: action_delegation_total(address: $address) { + coins + } + unbondingBalance: action_unbonding_delegation_total(address: $address) { + coins + } + redelegationBalance: action_redelegation(address: $address) { + redelegations + } + rewardBalance: action_delegation_reward(address: $address, height: 100) { + coins + } + vesting_account(where: $where) { + id + type + original_vesting + start_time + end_time + } }` let params = { - addresses: addresses + address: address, + where: { + address: { + _eq: address + } + } } let resp = await this.graphql_client.query(query, params); - return resp as Records[] - } - - async get_account(address: string): Promise { - const record = await this.get_accounts([ address ]); - - const { account, account_balance, delegations, unbonding, delegationRewards } = record.data; - - if (account.length > 0) { - return { - account: account[0], - } as Record; - } - - return {} as Record + return resp.data as Account } - - async get_total_supply(): Promise { let query = `query Supply { supply(order_by: {height:desc} limit: 1) { diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 0fa5f1d6..edfc092c 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,6 +1,10 @@ import { Request } from "itty-router"; -import { validate_cheqd_address } from "../helpers/validate"; +import { is_delayed_vesting_account_type, validate_cheqd_address } from "../helpers/validate"; +import { ncheq_to_cheq_fixed } from "../helpers/currency"; +import { BigDipperApi } from "../api/bigDipperApi"; +import { GraphQLClient } from "../helpers/graphql"; import { total_balance_ncheq } from "../helpers/node"; +import { NodeApi } from "../api/nodeApi"; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -9,5 +13,23 @@ export async function handler(request: Request): Promise { throw new Error("No address specified or wrong address format."); } - return new Response(await total_balance_ncheq(address)) + let node_api = new NodeApi(REST_API); + let auth_account = await node_api.auth_get_account(address); + + if (is_delayed_vesting_account_type(auth_account?.["@type"])) { + let balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); + let rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); + let delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); + + return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); + } + + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); + + let account = await bd_api.get_account(address); + console.log(JSON.stringify(account)) + let balance = total_balance_ncheq(account); + + return new Response(ncheq_to_cheq_fixed(balance)) } diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 644c56de..fa72debb 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,32 +1,26 @@ +import { Account } from "../types/bigDipper"; import { Coin } from "../types/node"; -import { NodeApi } from "../api/nodeApi"; -import { is_delayed_vesting_account_type } from "./validate"; -import { ncheq_to_cheq_fixed } from "./currency"; +import { ncheq_to_cheqd } from "./currency"; -export async function total_balance_ncheq(address: string): Promise { - // let balance = Number(record.account_balance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); - const node_api = new NodeApi(REST_API); - const auth_account = await node_api.auth_get_account(address); +export function total_balance_ncheq(account: Account): number { + let balance = Number(account?.accountBalance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); - const balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); - const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); - - if (is_delayed_vesting_account_type(auth_account?.["@type"])) { - console.log('is delayed vesting account') - const delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); - - return Number(ncheq_to_cheq_fixed(balance + rewards + delegated)); + let delegations = 0; + if (account?.delegationBalance?.coins) { + delegations = ncheq_to_cheqd(account?.delegationBalance?.coins[0].amount); } - const delegated = Number(auth_account?.base_vesting_account?.delegated_free?.find(d => d.denom === "ncheq")?.amount ?? '0'); + let unbonding = 0; + if (account?.unbondingBalance?.coins) { + unbonding = ncheq_to_cheqd(account?.unbondingBalance?.coins[0]?.amount); + } - console.log({ - delegated_free: auth_account?.base_vesting_account?.delegated_free, - delegated_vesting: auth_account?.base_vesting_account?.delegated_vesting, - original_vesting: auth_account?.base_vesting_account?.original_vesting - }) + let rewards = 0; + if (account?.rewardBalance?.coins) { + rewards = ncheq_to_cheqd(account?.rewardBalance?.coins[0]?.amount); + } - return Number(ncheq_to_cheq_fixed(balance + delegated + rewards)); + return Number(balance + delegations + unbonding + rewards); } export function delayed_balance_ncheq(balance: Coin[]): number { diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index 5d56a805..0a89cb56 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -1,33 +1,35 @@ import { Coin } from "./node"; -export class Records { - public accountBalances: { coins: Coin[] }[]; - public delegations: { amount: Coin }[]; - public unbonding: { amount: Coin }[]; - public delegationRewards: { amount: Coin[] }[]; +export class Account { + public accountBalance: { coins: Coin[] }; + public delegationBalance: { coins: Coin[] }; + public unbondingBalance: { coins: Coin[] }; + public rewardBalance: { coins: Coin[] }; + public vesting_account: { + id: string, + type: string, + original_vesting: Coin[], + start_time: number, + end_time: number + }[]; - constructor(accountBalances: { coins: Coin[] }[], delegations: { amount: Coin }[], unbonding: { amount: Coin }[], delegationRewards: { amount: Coin[] }[]) { - this.accountBalances = accountBalances; - this.delegations = delegations; - this.unbonding = unbonding; - this.delegationRewards = delegationRewards; - } -} - - -export class Record { - public account: { address: String }; - public account_balance: { coins: Coin[] }; - public delegations: { amount: Coin }[]; - public unbonding: { amount: Coin }[]; - public delegationRewards: { amount: Coin[] }[]; - - constructor(account: { address: String }, account_balance: { coins: Coin[] }, delegations: { amount: Coin }[], unbonding: { amount: Coin }[], delegationRewards: { amount: Coin[] }[]) { - this.account = account; - this.account_balance = account_balance; - this.delegations = delegations; - this.unbonding = unbonding; - this.delegationRewards = delegationRewards; + constructor( + account_balance: { coins: Coin[] }, + delegation_balance: { coins: Coin[] }, + unbonding_balance: { coins: Coin[] }, + reward_balance: { coins: Coin[] }, + vesting_account: { + id: string, + type: string, + original_vesting: Coin[], + start_time: number, + end_time: number + }[]) { + this.accountBalance = account_balance; + this.delegationBalance = delegation_balance; + this.unbondingBalance = unbonding_balance; + this.rewardBalance = reward_balance; + this.vesting_account = vesting_account; } } From 18720bb14d1db2db0bbe54b070b174f6012cb8b5 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 06:08:47 -0300 Subject: [PATCH 018/224] Fixed update balance per account endpoint (temporary) --- src/handlers/circulatingSupply.ts | 50 +++++++++++++++---------------- src/handlers/totalBalance.ts | 1 - src/helpers/balance.ts | 28 ++++++----------- 3 files changed, 34 insertions(+), 45 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 5c50d7e2..5abc1c14 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -9,42 +9,42 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P let bd_api = new BigDipperApi(gql_client); let filtered_accounts = filter_marked_as_account_types(circulating_supply_watchlist); - let non_circulating_accounts = await bd_api.get_accounts(filtered_accounts.other); + // let non_circulating_accounts = await bd_api.get_accounts(filtered_accounts.other); let total_supply = await bd_api.get_total_supply(); let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); - if (typeof non_circulating_accounts === 'object') { - try { + try { + const cachedBalances = await CIRCULATING_SUPPLY_WATCHLIST.list({ + prefix: "grp_1." + }) - const cachedBalances = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: "grp_1." - }) - let non_circulating_supply_ncheq = 0; - for (const account of cachedBalances.keys) { - const k = account.name.split('.') - console.log(`getting from KV: ${k[1]}`) + let non_circulating_supply_ncheq = 0; + for (const account of cachedBalances.keys) { + const k = account.name.split('.') - let cached = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${k[1]}`); - if (cached) { - const data = JSON.parse(cached) - non_circulating_supply_ncheq += cheqd_to_ncheq(data.total_balance); - } + console.log(`getting from KV: ${k[1]}`) + let cached = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${k[1]}`); + if (cached) { + const data = JSON.parse(cached) + non_circulating_supply_ncheq += cheqd_to_ncheq(data.total_balance); } - console.log(`Non-circulating supply: ${non_circulating_supply_ncheq}`); - // Get total supply - let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); - console.log(`Total supply: ${total_supply_ncheq}`); - - // Calculate circulating supply - return total_supply_ncheq - non_circulating_supply_ncheq; - } catch (e) { - console.error(e) + } - return total_supply_ncheq + console.log(`Non-circulating supply: ${non_circulating_supply_ncheq}`); + // Get total supply + let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); + console.log(`Total supply: ${total_supply_ncheq}`); + + // Calculate circulating supply + return total_supply_ncheq - non_circulating_supply_ncheq; + } catch (e) { + console.error(e) } + + return total_supply_ncheq } export async function handler(request: Request): Promise { diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index edfc092c..31ab6717 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -28,7 +28,6 @@ export async function handler(request: Request): Promise { let bd_api = new BigDipperApi(gql_client); let account = await bd_api.get_account(address); - console.log(JSON.stringify(account)) let balance = total_balance_ncheq(account); return new Response(ncheq_to_cheq_fixed(balance)) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index c4cc2336..0f726bcf 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,10 +1,11 @@ import { NodeApi } from "../api/nodeApi"; -import { ncheq_to_cheqd } from "./currency"; -import { total_balance_ncheq } from "./node"; +import { GraphQLClient } from "./graphql"; +import { BigDipperApi } from "../api/bigDipperApi"; export async function updateBalance(node_api: NodeApi, address: string): Promise { - const account = await node_api.auth_get_account(address) - const auth_account = await node_api.auth_get_account(address); + const gql_client = new GraphQLClient(GRAPHQL_API); + const bd_api = new BigDipperApi(gql_client); + const account = await bd_api.get_account(address); try { const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${address}`) @@ -13,24 +14,13 @@ export async function updateBalance(node_api: NodeApi, address: string): Promise console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) } - const balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); - const rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); - const delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); - - let res = { - balance: ncheq_to_cheqd(balance), - rewards: ncheq_to_cheqd(rewards), - delegated: ncheq_to_cheqd(delegated), - total_balance: await total_balance_ncheq(address) - }; - - console.log(`account "${address}" total balance: ${res.total_balance}`) + console.log(`account "${address}": ${JSON.stringify(account)}`) try { - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_1.${address}`, JSON.stringify(res)) - console.log(`account "${address}" balance updated. (res=${JSON.stringify(res)})`) + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_1.${address}`, JSON.stringify(account)) + console.log(`account "${address}" balance updated. (res=${JSON.stringify(account)})`) - return new Response(JSON.stringify(res)); + return new Response(JSON.stringify(account)); } catch (e) { console.error(e) } From 39583e452b3daf0aeaf419cd19686501bf1e19aa Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 06:12:59 -0300 Subject: [PATCH 019/224] Fixed balance query checks --- src/helpers/node.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/helpers/node.ts b/src/helpers/node.ts index fa72debb..ebd459b5 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -6,17 +6,17 @@ export function total_balance_ncheq(account: Account): number { let balance = Number(account?.accountBalance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); let delegations = 0; - if (account?.delegationBalance?.coins) { + if (account?.delegationBalance?.coins && account?.delegationBalance?.coins.length > 0) { delegations = ncheq_to_cheqd(account?.delegationBalance?.coins[0].amount); } let unbonding = 0; - if (account?.unbondingBalance?.coins) { + if (account?.unbondingBalance?.coins && account?.unbondingBalance?.coins.length > 0) { unbonding = ncheq_to_cheqd(account?.unbondingBalance?.coins[0]?.amount); } let rewards = 0; - if (account?.rewardBalance?.coins) { + if (account?.rewardBalance?.coins && account?.rewardBalance?.coins.length > 0) { rewards = ncheq_to_cheqd(account?.rewardBalance?.coins[0]?.amount); } From 2ec827f5a0d63e6ca53f83868ed37433c2ee515a Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 06:18:09 -0300 Subject: [PATCH 020/224] Fixed circulating supply endpoint to work with new query --- src/handlers/circulatingSupply.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 5abc1c14..affe411f 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,8 +1,9 @@ import { GraphQLClient } from "../helpers/graphql"; import { BigDipperApi } from "../api/bigDipperApi"; import { Request } from "itty-router"; -import { cheqd_to_ncheq, ncheq_to_cheq_fixed } from "../helpers/currency"; +import { ncheq_to_cheq_fixed } from "../helpers/currency"; import { filter_marked_as_account_types } from '../helpers/validate'; +import { total_balance_ncheq } from "../helpers/node"; async function get_circulating_supply(circulating_supply_watchlist: string[]): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); @@ -23,12 +24,10 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P for (const account of cachedBalances.keys) { const k = account.name.split('.') - console.log(`getting from KV: ${k[1]}`) - let cached = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${k[1]}`); if (cached) { const data = JSON.parse(cached) - non_circulating_supply_ncheq += cheqd_to_ncheq(data.total_balance); + non_circulating_supply_ncheq += total_balance_ncheq(data); } } From 9556b6193ac0dee754ec5d1322f5b9b3cf6d62ef Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 06:24:33 -0300 Subject: [PATCH 021/224] Fixed update all balances endpoint to feed KV (temporary endpoint) --- src/handlers/balanceUpdater.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index eaff1f5e..6bd51893 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -2,13 +2,14 @@ import { NodeApi } from "../api/nodeApi"; import { Request } from "itty-router"; import { validate_cheqd_address } from "../helpers/validate"; import { updateBalance } from "../helpers/balance"; +import { updateAllBalances } from "./cron"; export async function handler(request: Request): Promise { let node_api = new NodeApi(REST_API); const address = request.params?.['address']; if (!address || !validate_cheqd_address(address)) { - throw new Error("missing/invalid account address") + return await updateAllBalances({}) } return updateBalance(node_api, address); From 8cd17caa4ee891a717677310ab8b97e1cadec887 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 06:43:17 -0300 Subject: [PATCH 022/224] Better error handling so that API doesn't crash --- src/handlers/cron.ts | 4 +++- src/helpers/balance.ts | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index aa929ef5..87aa7e64 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -19,7 +19,9 @@ export async function updateAllBalances(event: Event) { } const res = await updateBalance(node_api, addr) - balances.push({ account: addr, balances: await res.json() }) + if (res !== undefined) { + balances.push({ account: addr, balances: await res.json() }) + } } return new Response(JSON.stringify(balances)); diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 0f726bcf..de070f58 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -16,15 +16,14 @@ export async function updateBalance(node_api: NodeApi, address: string): Promise console.log(`account "${address}": ${JSON.stringify(account)}`) - try { + if (typeof account === "object" && account.accountBalance) { await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_1.${address}`, JSON.stringify(account)) console.log(`account "${address}" balance updated. (res=${JSON.stringify(account)})`) return new Response(JSON.stringify(account)); - } catch (e) { - console.error(e) } } catch (e) { console.error(e) + return new Response(JSON.stringify({ error: e })) } } From ac8391e0cae8a345b014d30e70b78219e6d36a97 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 06:59:29 -0300 Subject: [PATCH 023/224] Fixed all balances updater --- src/api/bigDipperApi.ts | 3 +- src/handlers/balanceUpdater.ts | 12 +- src/handlers/circulatingSupply.ts | 289 ++++++++++++++++++++++++++++-- src/handlers/cron.ts | 16 +- src/helpers/balance.ts | 17 +- src/index.ts | 32 ++-- wrangler.toml | 4 +- 7 files changed, 328 insertions(+), 45 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index d3baf4bf..86caeb0a 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -20,7 +20,7 @@ export class BigDipperApi { redelegationBalance: action_redelegation(address: $address) { redelegations } - rewardBalance: action_delegation_reward(address: $address, height: 100) { + rewardBalance: action_delegation_reward(address: $address) { coins } vesting_account(where: $where) { @@ -56,7 +56,6 @@ export class BigDipperApi { let resp = await this.graphql_client.query<{ supply: { coins: Coin[] }[] }>(query); - console.log(resp) return resp.data.supply[0].coins; } diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index 6bd51893..71ba8133 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -9,8 +9,16 @@ export async function handler(request: Request): Promise { const address = request.params?.['address']; if (!address || !validate_cheqd_address(address)) { - return await updateAllBalances({}) + const grp = request.params?.['grp']; + + console.log(`updating all account balances (group: ${grp})`) + const res = await updateAllBalances(Number(grp), {} as Event) + if (res !== undefined) { + return res + } } - return updateBalance(node_api, address); + return updateBalance(node_api, address ?? ""); } + + diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index affe411f..2ad4643e 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -17,20 +17,28 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P try { const cachedBalances = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: "grp_1." + prefix: "grp_" }) + console.log(`found ${cachedBalances.keys.length} cached items`) + let non_circulating_supply_ncheq = 0; - for (const account of cachedBalances.keys) { - const k = account.name.split('.') + outer: + for (const account of cachedBalances.keys) { + const k = account.name.split('.') - let cached = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${k[1]}`); - if (cached) { - const data = JSON.parse(cached) - non_circulating_supply_ncheq += total_balance_ncheq(data); - } + for (const n in [ 1, 2, 3 ]) { + let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${n}.${k[1]}`); - } + if (cachedFound) { + console.log(`found cache entry: ${cachedFound}`) + const data = JSON.parse(cachedFound) + non_circulating_supply_ncheq += total_balance_ncheq(data); + continue outer + } + } + + } console.log(`Non-circulating supply: ${non_circulating_supply_ncheq}`); // Get total supply @@ -49,7 +57,268 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P export async function handler(request: Request): Promise { let addresses_to_exclude: string[] = (await CIRCULATING_SUPPLY_WATCHLIST.list()).keys.map(k => k.name); - let circulating_supply = await get_circulating_supply(addresses_to_exclude); + // let circulating_supply = await get_circulating_supply(addresses_to_exclude); + let circulating_supply = await get_circulating_supply(getTempAddresses()); return new Response(ncheq_to_cheq_fixed(circulating_supply)); } + +function getTempAddresses() { + let addresses_to_exclude = [ + "cheqd102mzdp87r6tjy0ttlgsxmldjex68k9tuxnqkwn", + "cheqd1074lwwsya2nvgfklg403aa9jtkgl7nml5zylg9", + "cheqd108q99w63he0uq2xa28gxf7hkpeqt8r5uqt9ghk", + "cheqd109c4836wy0aa9ft7uhvu6z5yxx55zh3yqjfle0", + "cheqd10e4zkalt6uqd3cwgtr8g4ychjjhrwq5n743j65", + "cheqd10lffy494en7teacp7ty9lgj5m9u333ps8cjns6", + "cheqd10nse74dkac20enn3ame36u73llc2tmnuehpj5f", + "cheqd10pps0c8msgsvjzrx2zge0tl68x93dzzfyszhqq", + "cheqd10tu4wnj67d8cnpwycllwur6qnltg87xfshndpf", + "cheqd10yut7vc3uvrasju4aa6tpwdxrmwdn6auxxeqft", + "cheqd128fra0sj9fgz9pmupc7dnrupgr38nm86wq3r34", + "cheqd12ht2skvkw4nnhq6fc48vlkugl0myryymaxmhhh", + "cheqd12lvv4cqdznu4q4gnwm3x4tjfqa4de49v49quq2", + "cheqd12tt8jn5fdlvedmxn4356rvn362g77tz4yv5za4", + "cheqd12vdx3selza7mk0qw6njtupfmkfk4nzl2jyqac7", + "cheqd12ys2yeqwa57083xhd5wjgjjp69f7jafqd6q4zr", + "cheqd130g8d4t7x5vpkdvuekzltghcv7re6ul35dgcq2", + "cheqd13duhx9h5g9z7qxy2u0uzselv2hljmzzslae5l5", + "cheqd13tgpmr4ueuw022l6nxkew823czawc00j56uh9w", + "cheqd13w44vy7s6mrqpxwc5ujpjfcldjzhkx5xfzeq94", + "cheqd13wvtjk3v5g4q7pema74nv9lrarr8e6vlx93zat", + "cheqd13x8zvv4a36mpcp278m53r5wesfsx2wwsrsn6kn", + "cheqd144j9sw5884vwnlem6sedtzj8rgsmaz7zsa6r96", + "cheqd14av5en2qpj3fgzyt2krx9kt3mlp8cn8mzqppfs", + "cheqd14c9fyc55szuwep6y2krnrqvm7qs97ret3uyjps", + "cheqd14gxh72xdtx5hh7ctg70lg3yy3gratlexsaj8ze", + "cheqd14hj6lrj3eaqlg0syf642eng3a2qm743kdaaddp", + "cheqd14re975jg2e3q2r60e0xdag77n8mz8t0mpffe9z", + "cheqd14ucjrgm8jm5z7yv322lgz3w9wz2psve7wncv5q", + "cheqd14vnqa3msv72ucp4j3lqhpvsmxuyu89qzvlazpk", + "cheqd14vyyr9a57mspj4kugg6yylcuqkxytm2q4srzra", + "cheqd14y0hce2rclpmcn8w0vvffc4f8fjlghduzn09pn", + "cheqd152wh3482jks8me22l085c7jcxtk74rm9yrhv08", + "cheqd157kn6s6p74ps7kmgnr6vvrqj47sx2895mxvr0f", + "cheqd15dg3zjxru3pus6qu5spn4g7n8d6f68wtslvssn", + "cheqd15erqx74e4q532cxp9jgsqc5gc045kaefnuakav", + "cheqd15eu3jgyezmct6yhanh3dpwsfn9qrulkna7q4rx", + "cheqd15gjydlputcmwxudnasc6c6sla2zy0cjs88svyd", + "cheqd16qcyenxplnsrgrpnnqmxgv2p2ddqwnjpz9quy3", + "cheqd16umet2xymlj8xtnqzr34vd06sml3fz8ujlm5en", + "cheqd16xz82c9u3w3yfsx6kjrrj39v5nx2lflysyf5ay", + "cheqd16zs74amn0es66vrryevr3pfa25pjwyllv575jv", + "cheqd173k83wpkgtr6xtx26caumsqkpwplpt0k0vr0y7", + "cheqd1774h8jzpv2kk5pasdrumknauypnm05nm6qjcsv", + "cheqd1782hcsf7m88pswfk2kn3zeujjzuykgzs5f00j8", + "cheqd1786x2uuudf5clle7s4e6y3safl4hwkhtgymdny", + "cheqd17c4jf69dl9d38aw5uysn8clf95xs8t74w33m6z", + "cheqd17jhrply2h0xk0zwku6859dr0y693379tunwcnl", + "cheqd17l9yrd0gt2dkgqdu4nknwdr7g7c22y4k94s2pl", + "cheqd17lqrgwg8lx8hp69jhku3t4hxx86qpw5cgx5l4n", + "cheqd17p5tyat3nr2kyqt49rak05s8ceftq6ykx5qt4t", + "cheqd17wrwqxsfwy4rqlltjhj6jxtz68tvxm0ykge5dr", + "cheqd17xekaxgwvn04dqmda37n5v5yatwe0pwm6umw2j", + "cheqd17xh3a5qvqy3tp4y9hp482ssl3pe2gq9j5m8zmh", + "cheqd17zjfjw5emz9r48wtma2hwhql334s0f375jvx72", + "cheqd182d4zh33fmgj38yhqv0ryapuancagvpj52frye", + "cheqd187k4vzggkx0kdurxg3xffdeehnfw4m0hm7h27s", + "cheqd18e9am44nt43w4qdg3s402ymzrvfdc37qd3pyt9", + "cheqd18jzpf9mvpaprtccvvsvpunv758p4avkk62nvd9", + "cheqd18n4a450pwne88u8g5p53pufdyaqhlxes6v5hwf", + "cheqd18p5t6pru4nnh8qmsnf3w90uqvk2e9aezlhym53", + "cheqd18pcq5jxm8vz0jkmm484y7n6g49ysclkz5jwr2e", + "cheqd18pwcnw73tsg3gguukwgx50u2d62zk4hspv8l32", + "cheqd19cjct09g4a50yqjavuflscqfqcqns2yp05n5ms", + "cheqd19tql8zv4kpp7umgx0nn6nt7qf0k29kx4h4trpe", + "cheqd19wxf36lwgkktuampg3w6uaw7grsy3gnrxj4t4h", + "cheqd1a25g2fln0racf9dlfn3g6e3vezvgynnez8uv0z", + "cheqd1aasqs6cn3hy2pqsypd6az0y8f5va32hhpxz8gh", + "cheqd1avu2ll646ypnjqa542rqpe0g26cfesve337ml8", + "cheqd1awfupskqfhev874vfjnpw9t0v94uuqhk74q3d4", + "cheqd1c3kwrlxsy3yz3phpap20z20j5qr2du3clgzs30", + "cheqd1c6g9nnlw2ht38ak3cpxu7p4yjaeef8m7frc0xp", + "cheqd1cgmzu9f9heas5ta46drxpeuk5yyn8v36yjlje7", + "cheqd1ck6rv3mst963m3pm5k8rg66elera8v26pah03f", + "cheqd1czauh9jmwgh9yc090gnhvmk337kz3jnarwuxed", + "cheqd1ddthz5uhx2382av5ew9t7yx25dznwhy0rpzzz3", + "cheqd1dksptzr757fe9tl23ywmp4guucsz0fl8e8ggll", + "cheqd1dp2tqng7lkm094t0g85etjet5rvlcue6hynx6p", + "cheqd1dqql2afmdc5wwzl74mu6f3jg5fxk6ksy6z7ma8", + "cheqd1dvxce4elk8lfph8c52mlevgkyg5lxdw6vgjf8h", + "cheqd1e4tdx3xfchd5vj0tmqgtdpkmlsftqvej8j9l6v", + "cheqd1e6y9pqk538qwdsra4y44a3qjll9tmgx44tx8e2", + "cheqd1e7jpqg3960msznsghctj9cxv4w7w4qph6g8jlz", + "cheqd1ecxmdrr5fhghskvawdyulqrczm604kx8dr5lgg", + "cheqd1ekt65hyudh5lhzjw55ctxqgtkddjeykfcqljkv", + "cheqd1eld0vlqy3tlmkhf7lgdcefc8qz7ny0cdk9j2ry", + "cheqd1ereqx3s4c3grc78w8ta2nmjk7a35p694rkt4zc", + "cheqd1f38cesgyn8yhkl9lhrxv4urr9gtl444k02uxac", + "cheqd1f9aunuufxy6euayzegxvp4jy3uy2xzcuarnmsl", + "cheqd1f9xhnvmdetn9mkh0w3pcll9rzy90uhwj5c0k9k", + "cheqd1fkxggf3m5ptap7h7fyh9ht095khuar026crx3c", + "cheqd1fqfwhraax5ngk0upwknej0jlpxc3ev7me7ht7s", + "cheqd1ftt2485zh3yn0t850atw34m2ctwvghsyf6kdr6", + "cheqd1fww87qsvqchyq9xwqcj5w550f2kxu6eyf75wsu", + "cheqd1fxkzl28rqsmjrl48qhkrzug42fhwg4xxtyuvgq", + "cheqd1gal47c74jets5ref6wjxguzqmkgry73z4u8tdh", + "cheqd1gp2nftpztxh44ex3vjqwfdvnvj2fkwtpn3g4gg", + "cheqd1gq6sn7fdeeplltyuv0tp76hq7766v64w8yktxj", + "cheqd1gzzlq30nwlctxdjg36fmy8djjycs2puvmmnmke", + "cheqd1h6hktd76fa93zt078nlrcrdndk7ynmg04an0fh", + "cheqd1h8mxrwh2n9gklz52m3wx0zr9230m727cjrsxw9", + "cheqd1hmc79es4m72l6yy4dmya4dclhnzdp3550ykfy8", + "cheqd1hmhktc89jmzhyf9y6qt7dzc7cy7cgu3cnqs6wp", + "cheqd1hn9m0pjxp4p4003xnzcmze7gydhg8a8r0858vd", + "cheqd1hprhwj8cny5p3w2nt9av3w0d9kr80ttdhagjte", + "cheqd1hzslf2k0a7k722vzvamyrt7tfq3447xk5ggfvg", + "cheqd1jcm85acd9jvvz6u43v3f2pk9lncatu879c4a8u", + "cheqd1jhttw93wzjrcjm8a74g8aph3wnpjfnqd9568t6", + "cheqd1jp00saqjfcdxkxqf9e8vrpyd4at9xlc7gf2syg", + "cheqd1juq5zgzjr92c4wn8cy39g3sws28du0crmfpjuc", + "cheqd1k86579yh9j49q9xtp4ta55zy6ad9azhrraal7k", + "cheqd1kj93ee7zkw44p9h0fnd6f8yhzeh509wg4acjjn", + "cheqd1kn3vj97l4k8t55073ragpk9zeufrnzh80ah9vs", + "cheqd1kputn5c3f0hn7da98lqf4c35fllxxeycrldutr", + "cheqd1kqx7q0dr3pn8hyxpu25gaz3m3qs835ldklmumk", + "cheqd1kr7dv9n6dnmjy0dldqxtp5dgq0jh6uhd5hx9z2", + "cheqd1kv5fha5ylprfd558f7r0uyex8rek8etpfxc6wu", + "cheqd1l8hvrw2j82er654s3dvvj9w9jjkcekqsx0jz8w", + "cheqd1l8uy4yn6utfgy65smmw40m8c4uyarw2hc8pkw9", + "cheqd1la5t260ggu3mkc7aha3ftwj40laynygy9chevy", + "cheqd1lg0vwuu888hu4arnt9egtqrm2662kcrtkqz49j", + "cheqd1lhch7spf87rgnev56xnqh2dqhpfk3eatdnth5y", + "cheqd1lqtfptxeurmd54xy7zkns7qwwutvns0w4ufyft", + "cheqd1m6uas09cczhyra3up3tn4calwvl7ygxvxym7a0", + "cheqd1m7d75z29hqu2d0uv0mk2s88um5hvj259md7758", + "cheqd1ma754mlx2rdvshae9t8jkz2c4x7ft2lpkmn8vt", + "cheqd1mjlhlt3yx9tqjsnprtsxemf9hwxwxgd3r6xrnr", + "cheqd1mw6xpmqt7u7mlcu0lerh7dw5c9rvsqsthqcgrw", + "cheqd1nfsz8r4p359fsctk0lt4ee3kma0zg89xldxpqu", + "cheqd1nqzypqthx348llfuj9j5xftzfua2l75qy9fwq6", + "cheqd1nu5aaszuxjvxt5x8eq2s8tlad4nwjk2acn8cj3", + "cheqd1p02e0x204a49utmkgwjdk9kpy8gqtcm7mpu87l", + "cheqd1p9szkzftfddxce82ymckgxuxwtqc6d4k2at4ep", + "cheqd1pc02hdy79wuwyxzy7ucumnmaexp9gx5wa5ckqr", + "cheqd1pdu0r224x6tvfkhswnqrq9z5w4d73jns4u3k2m", + "cheqd1phmz70wgzyahlacn78hwzdkwu5fky5tfl8gfs3", + "cheqd1pqqlxl3msedxcej57q9examfzhu0ttzdsp02pw", + "cheqd1psrm6t3hqwzvmfnw6x4ysf0hd240a3k3d9yy27", + "cheqd1py8mmqqv3wk22ae4pz0r0da7x9zzrxefzugklh", + "cheqd1pydlf8h7ezlkpwq8ud3gdx83ra9me47z48qtat", + "cheqd1q3k2ne32vtx2dvf8ee7vpafcvdsrsvqx5thf72", + "cheqd1q449p4s8c4t8chdrx9gk5nlqt43v508erl7qsu", + "cheqd1q5cazs6yuvmcf34z369z2fvt2ec6n2f7vp06hm", + "cheqd1q8ghndxp657hajzsk4trfkuh5zrm74vedyhplh", + "cheqd1q9tznprszw5usu55yfk42fenfe6tyq58j53skg", + "cheqd1qaa9pd60g6ry9u3kf9sezw26lyejky8xayr52s", + "cheqd1qc7nexzm8egzte8amh0qusg99xjsnc2w7m96c5", + "cheqd1qn78q6tddxxz2rpcflsuququvuvjalrunxp3c7", + "cheqd1qs0nhyk868c246defezhz5eymlt0dmajna2csg", + "cheqd1qvj2uuyvalx22lqnehh3awjcq35a0rz5gpudtm", + "cheqd1r2g7e6ymdwfql5j87gs0xv5llh9xp692vfwruq", + "cheqd1s3hqzaraajvlkpl4fseteenad2zev9hmhfdhrd", + "cheqd1s7qgtdpyw9cj2swkdhzywwlssxapndcalrfy5q", + "cheqd1s8f3rqprpwhn0vxl6me76qwg6y9cpxj2yjjy3l", + "cheqd1s9wn3wxea4maef8t04jyusnpc29p58v83eesz0", + "cheqd1scxe3jqyuqv0flgl0w3s95hhy9jsp98h5a24pk", + "cheqd1sr0066h5p2ncfc8hnrgqlw3lqxgylwn06ccl9e", + "cheqd1sv4w8tadeulklgwpu477ls3rvd2se3cuftdvkr", + "cheqd1t0qhk7kqe6vv36alehrwn2ht5e0dc8wh2uzkr6", + "cheqd1t0vw4kha8ffw0sfsd2k56287q73zwnca5gr7l2", + "cheqd1t3gsdl2uux0rmt3acn8qursc9sarhtm6mpa3hq", + "cheqd1t5pl7x25egk2xcfu2kyg4a7ga4hyg2g5anv5hh", + "cheqd1tgtgmksy2de42nzahk9kwml9crf36r735xwazj", + "cheqd1thhqk2q4s8h3ugwjm4h2v09gxaturfzkyjeux3", + "cheqd1tml0chnp3mee88cyk63la3ctj9ndu4vtl2e2m7", + "cheqd1tpvhkuvn8je2qdghexfsgeg0q37zcqej06m30x", + "cheqd1tqpxjdnxcjqswrcq5slz64wrh7t5j6jle7gwfh", + "cheqd1tt5p2xwz3xv2d4wjkegycu6pauv3jxyta3vj0r", + "cheqd1tx65lajxvzwmvfxwvev23kxac2v3zj3jg0hrcw", + "cheqd1u3uwj2rvxt6ytweypemcc35vy80zwlnx7ly6wt", + "cheqd1ucxsrv5yyvxhjk2cu3gl54l94dru7apgwrlk9q", + "cheqd1uvjqw3wadfjycznpe0z8u36aqnecndtzlwgf7s", + "cheqd1va3slq5k97a008gxncn9eavvf4mt84xcmej65l", + "cheqd1vex9phvwcwr785k2jal7tv2pklysqx94kzay8a", + "cheqd1vh0rar0jgyx8xeett26nv77jnvxr99k8q8qa6q", + "cheqd1vj9qtf9fy00q8xnwxfwsswtr24ajl8a592hyp8", + "cheqd1vl23k3x73wzq4t7x3sh54n5f40kpse32pleka6", + "cheqd1vlu2azc2fs9p5zffyeefth0l79eygwj8vmt2r6", + "cheqd1vusv4s233mw867ua63m7upjxzgcyqhwhumqe6d", + "cheqd1vxfyppcnu6lz72mdc706mlv7kx8434jaknhgt6", + "cheqd1w5xsmev2zv59yhtk5nsl6xrdwpu588k5su6xs0", + "cheqd1wsf8ns8gmcq76nes484pt46c9ru9c9mdmwm7sc", + "cheqd1wwwncrnde9myewpn8j00xeyt4hfs3sqmkyasqm", + "cheqd1wznaxsqya0meesp9f8xrknnreuszfd8kup3mq5", + "cheqd1wzyfq9ujg7vp3whn7nh7w5ner8p7ghk2pxw0g5", + "cheqd1x7ku9c2x2j263yaj48ct9nwjark75f0hhgmpvg", + "cheqd1x9dc5enk7f77apgscc3emep5nz0fapxm4w9r9m", + "cheqd1xqgm3nw0uh4k4zj0u4l2ljfqslrfumff5syu49", + "cheqd1xqxz5wneg2hqg9tugzuv8v8wgyr2pl32v2h3hl", + "cheqd1xs7kvk7n7m6ca6yw7ysxxfulwauc2s46z2w283", + "cheqd1xusdhhtlqk0vetgqg7nu39j9q62zq63q22k463", + "cheqd1xwnu2mm96uexuergqk0kqt4ds84x9zsjyne49e", + "cheqd1xyyygaxx384k07sv3rk9zj7dezuzf250pahr0w", + "cheqd1y6ekzultqshf8xwtv46mlzfquj6ys3vmncascq", + "cheqd1y8edvwl536rxuqdznh48mh6mg648gf9a3xx2s9", + "cheqd1ygwl8m54cpenq7t5v5p0y9xau3gmg873fqlqag", + "cheqd1ylfukxalf0zr2wdma0gx33e9fccha94qr59rj4", + "cheqd1yp52vx7sjv2dhhfqe2n52ntr0av39lye9fx5gl", + "cheqd1yqj0s0htaseqc9prkcsmucme3ef5fsyq5mkqrd", + "cheqd1zh8g8lw0zhp0qk0fxjq9n5p2yjc4xgpca5gzds", + "cheqd1zhqu2pzpsv0x5afmw3tm5579pqxy59msh8ytj3", + "cheqd1zhw4qtrd04kp7zz329zjlwy0s3vjunxua6rrqd", + "cheqd1zm2pz9gj4vsl278vydapx0x8e3fguvxalg7n98", + "cheqd1zmxl82zfgqsc0efuyhateaen8mzp7ngkaw4qxz", + "cheqd1zu4mm0lzhm7az4fwc6d28pkt2vvpte5ye45rgv", + "cheqd1zym5ync279m2sxx99dhtv4s00apr08lzw2g8g8", + "cheqd1zz0gck564g3dvaxctccc5kgxw2sgtes7p2led6", + "delayed:cheqd10vmxqqnc9j7axh808ymyz8t8kneux5n8xmmhq9", + "delayed:cheqd14c5zvsf74uayyp90mav49gx2aj250s8u8yv7y9", + "delayed:cheqd15yd3fnn3ev5xeacazejgejd5jxrnpusg8mgu8h", + "delayed:cheqd15zw9j5e3l4l3k6fex8e7hznzmf0mpz5vcwlrt3", + "delayed:cheqd17n6mefp8vhyrtvavc9fd9hg0hx624skgdjtrpz", + "delayed:cheqd184z8z58dmax77csg0a9tneku0sqskeqymx5hxw", + "delayed:cheqd18jc49zfkew8aj8zmxjyrk3nr8asqu8jxg56fwz", + "delayed:cheqd198yncktukks0aqtyk4vq0hw20v67ffr0rm8cla", + "delayed:cheqd1a0vumnau34trnvjpnqr4ywwaqd89l4s7ytwkk5", + "delayed:cheqd1ajefdpfng3l66gr2j3ygvh6efrr6twrd3968rk", + "delayed:cheqd1c5wva37ufnpdypxkxeya9kawrzgay6evnra50u", + "delayed:cheqd1cpj9vd5e3dyvyj0nds3hanz30zafakdc4h00wj", + "delayed:cheqd1crws5sz8tdl37v40tjn0lmlu6ztjhxar9jklqz", + "delayed:cheqd1dy69x2g8vj0sdnutr29lt2wqgxqw98pcxltuu5", + "delayed:cheqd1eraud0nswfglksl9tl8e5dlemrq5eualjftv3z", + "delayed:cheqd1ewhshdq9cdhxwx8vrnnffhfe8qw3szryqv4gu7", + "delayed:cheqd1exe54lymn5mhpjs0vj6yppnecdmtfss2qcd4cq", + "delayed:cheqd1g3379fkvfc6jgxa66dswzsmzeguryv6zkxca87", + "delayed:cheqd1gz3suxvpqrdwntnn7r95eerwa3ura92kp4tm8c", + "delayed:cheqd1jkzd7sn6dzn54tl0y4ztw7n68r0p4aw53f5mqx", + "delayed:cheqd1k3vc8xplwtqpyt55ysx085lhr09xxvhuq9ddra", + "delayed:cheqd1kaascflyltj3pp54hwarnat6kmwtw2zvllsqdc", + "delayed:cheqd1kq5atufq4ryhdld8s7cmy2735uxhh6wdakj9cj", + "delayed:cheqd1kysvc7hq8944jq68yjg7hqxud66r3ajx8uv5s8", + "delayed:cheqd1la5de5x52qw8vspwj70g6c6mem02al8sf4z4sm", + "delayed:cheqd1lp0lq9mhc22y42znga5mvjngjpdj4pgvgzs950", + "delayed:cheqd1lvgq858vk9hs9stfkkv0dhhvnwv5eczjx4rkjw", + "delayed:cheqd1m5r4emnpmgnl0xgptk4f4pkrqmnlrzpnhs8aqu", + "delayed:cheqd1mn83wm6f43f9ls6lsju8w2x4z4rg4rcvmcsq7z", + "delayed:cheqd1nn7rcyvfd3zqhumuwwj05wvdh4x28f2qs3sex5", + "delayed:cheqd1ppuya3ezvs2jq7n2e0ygec6r3uhx0gea6hl9vp", + "delayed:cheqd1pw0z58t0jrtexp26694ycrdgv6ycmt5fcy00vg", + "delayed:cheqd1q4lkgp3psk2fdxt2pgr0lujyj5fk4lz2wlz8dy", + "delayed:cheqd1q6wwcqf902ygu8z7dcyhh8ca0u6a5g4kns6he8", + "delayed:cheqd1qxr4jdgug8dve9kq6vc6pwdhpuyvlsql85xx0z", + "delayed:cheqd1r2y0ffh5xgkrnzelygz35mprjejaatnah9untf", + "delayed:cheqd1s0sngupwnkfv84r079x5ct74j8nnxgjfp89p54", + "delayed:cheqd1ssge5ykw4hvu4jy6eeen7hyxnw72mds6mzj79h", + "delayed:cheqd1ssjkeq5k385fydn9n7qvcp74pqkxgl8r8eczfq", + "delayed:cheqd1sueav9pmjy5k8crpg2nuakyssd86kcrrlc0qre", + "delayed:cheqd1t62uzdkyaw27v59ufyftzccl707k8a6rt9kcgn", + "delayed:cheqd1w53yz6ylpraykng3sw6r8lunuar7lteavel3x0", + "delayed:cheqd1x6srspstwx3pdta64d99en9radmaztms6lj0te", + "delayed:cheqd1y3fvu2ychq7247fkwfna0nn6exl9w6u009tnh7", + "delayed:cheqd1z8hvr2drwgnfqudkrqhwvdv5p40h8maw7jg92g", + ]; + + + return addresses_to_exclude; +} diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index 87aa7e64..2b087aec 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -1,26 +1,30 @@ import { NodeApi } from "../api/nodeApi"; import { updateBalance } from "../helpers/balance"; +import { Account } from "../types/bigDipper"; -export async function updateAllBalances(event: Event) { +export async function updateAllBalances(group: number, event: Event) { let node_api = new NodeApi(REST_API); - let balances: { account: String, balances: {}[] } [] = []; + let balances: { account: String, balances: Account } [] = []; try { - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${group}:` }); console.log(`found ${cached.keys.length} cached accounts`) for (const account of cached.keys) { let addr: string; - if (account.name.startsWith("grp_1.")) { - const parts = account.name.split('.') + if (account.name.startsWith("grp_")) { + const parts = account.name.split(':') addr = parts[1] } else { addr = account.name } + const res = await updateBalance(node_api, addr) if (res !== undefined) { - balances.push({ account: addr, balances: await res.json() }) + const data = await res.json() as Account; + console.log(`updating account (grp_${group}:${addr}) balance (${data})`) + balances.push({ account: addr, balances: data }) } } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index de070f58..c35886bb 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -10,17 +10,18 @@ export async function updateBalance(node_api: NodeApi, address: string): Promise try { const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${address}`) - if (typeof cachedAccount === "object") { + if (cachedAccount) { console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) - } - - console.log(`account "${address}": ${JSON.stringify(account)}`) - if (typeof account === "object" && account.accountBalance) { - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_1.${address}`, JSON.stringify(account)) - console.log(`account "${address}" balance updated. (res=${JSON.stringify(account)})`) + if (typeof account === "object" && account.accountBalance) { + const grpN = Math.floor(Math.random() * 3) + 1 + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${address}`, JSON.stringify(account)) + console.log(`account "${address}" balance updated. (res=${JSON.stringify(account)})`) - return new Response(JSON.stringify(account)); + return new Response(JSON.stringify(account)); + } else { + return new Response(JSON.stringify({ error: new Error("something went wrong") })); + } } } catch (e) { console.error(e) diff --git a/src/index.ts b/src/index.ts index 977e9f92..ede1d923 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,28 +12,30 @@ import { handler as balanceUpdaterHandler } from "./handlers/balanceUpdater"; import { updateAllBalances } from "./handlers/cron"; addEventListener('scheduled', (event: any) => { - event.waitUntil(updateAllBalances(event)); + event.waitUntil(updateAllBalances(1, event)); + event.waitUntil(updateAllBalances(2, event)); + event.waitUntil(updateAllBalances(3, event)); }) addEventListener('fetch', (event: FetchEvent) => { - const router = Router() - registerRoutes(router); - event.respondWith(router.handle(event.request).catch(handleError)) + const router = Router() + registerRoutes(router); + event.respondWith(router.handle(event.request).catch(handleError)) }) function registerRoutes(router: Router) { - router.get('/', totalSupplyHandler); - router.get('/balances/liquid/:address', liquidBalanceHandler); - router.get('/balances/total/:address', totalBalanceHandler); - router.get('/balances/vested/:address', vestedBalanceHandler); - router.get('/balances/vesting/:address', vestingBalanceHandler); - router.get('/staking/delegators/total', totalDelegatorsHandler); - router.get('/staking/delegators/:validator_address', delegatorCountHandler); - router.get('/supply/circulating', circulatingSupplyHandler); - router.get('/supply/staked', totalStakedCoinsHandler); - router.get('/supply/total', totalSupplyHandler); router.get('/', totalSupplyHandler); - router.get('/_', balanceUpdaterHandler); + router.get('/balances/liquid/:address', liquidBalanceHandler); + router.get('/balances/total/:address', totalBalanceHandler); + router.get('/balances/vested/:address', vestedBalanceHandler); + router.get('/balances/vesting/:address', vestingBalanceHandler); + router.get('/staking/delegators/total', totalDelegatorsHandler); + router.get('/staking/delegators/:validator_address', delegatorCountHandler); + router.get('/supply/circulating', circulatingSupplyHandler); + router.get('/supply/staked', totalStakedCoinsHandler); + router.get('/supply/total', totalSupplyHandler); + router.get('/', totalSupplyHandler); + router.get('/_/grp/:grp', balanceUpdaterHandler); router.get('/_/:address', balanceUpdaterHandler); // 404 for all other requests diff --git a/wrangler.toml b/wrangler.toml index 5fdaec9f..d550c5bc 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -34,7 +34,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa [[kv_namespaces]] binding = "CIRCULATING_SUPPLY_WATCHLIST" id = "ce2e547de27f467c86afa67996b8784a" -preview_id = "a9b6ebe3d1b44671acafb0ed5b7f8919" +preview_id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` @@ -88,7 +88,7 @@ binding = "CIRCULATING_SUPPLY_WATCHLIST" id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" preview_id = "1e1032cbf6854d88b12317da8a792928" -crons = ["0 * * * *"] +#crons = ["0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 58ed4ea383cd04bcf79380b9d338e12521125673 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 09:45:05 -0300 Subject: [PATCH 024/224] Only updating missing balances (temp) --- src/handlers/cron.ts | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index 2b087aec..5f02eb45 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -10,21 +10,30 @@ export async function updateAllBalances(group: number, event: Event) { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${group}:` }); console.log(`found ${cached.keys.length} cached accounts`) - for (const account of cached.keys) { + for (const key of cached.keys) { let addr: string; - if (account.name.startsWith("grp_")) { - const parts = account.name.split(':') + if (key.name.startsWith("grp_")) { + const parts = key.name.split(':') addr = parts[1] } else { - addr = account.name + addr = key.name } - const res = await updateBalance(node_api, addr) + const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key) + + if (found) { + const item = JSON.parse(found) + // only update errored for now + if (item.balances.error) { + const res = await updateBalance(node_api, addr) + + if (res !== undefined) { + const data = await res.json() as Account; + console.log(`updating account (grp_${group}:${addr}) balance (${data})`) + balances.push({ account: addr, balances: data }) + } + } - if (res !== undefined) { - const data = await res.json() as Account; - console.log(`updating account (grp_${group}:${addr}) balance (${data})`) - balances.push({ account: addr, balances: data }) } } From 871602e7ecfff8f1373956bc3d6b8de89242060f Mon Sep 17 00:00:00 2001 From: drgomesp Date: Tue, 27 Sep 2022 10:37:10 -0300 Subject: [PATCH 025/224] Reactivating cron --- src/handlers/circulatingSupply.ts | 300 ++---------------------------- src/handlers/cron.ts | 31 ++- src/helpers/balance.ts | 2 +- src/helpers/node.ts | 9 +- src/index.ts | 6 +- wrangler.toml | 2 +- 6 files changed, 38 insertions(+), 312 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 2ad4643e..12323f10 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -2,43 +2,33 @@ import { GraphQLClient } from "../helpers/graphql"; import { BigDipperApi } from "../api/bigDipperApi"; import { Request } from "itty-router"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; -import { filter_marked_as_account_types } from '../helpers/validate'; import { total_balance_ncheq } from "../helpers/node"; async function get_circulating_supply(circulating_supply_watchlist: string[]): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - let filtered_accounts = filter_marked_as_account_types(circulating_supply_watchlist); - // let non_circulating_accounts = await bd_api.get_accounts(filtered_accounts.other); - let total_supply = await bd_api.get_total_supply(); let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); try { - const cachedBalances = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: "grp_" + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ + prefix: 'grp_' }) - console.log(`found ${cachedBalances.keys.length} cached items`) + console.log(`found ${cached.keys.length} cached items`) let non_circulating_supply_ncheq = 0; - outer: - for (const account of cachedBalances.keys) { - const k = account.name.split('.') - - for (const n in [ 1, 2, 3 ]) { - let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${n}.${k[1]}`); - - if (cachedFound) { - console.log(`found cache entry: ${cachedFound}`) - const data = JSON.parse(cachedFound) - non_circulating_supply_ncheq += total_balance_ncheq(data); - continue outer - } - } - + for (const account of cached.keys) { + console.log(`looking for account: "${JSON.stringify(account.name)}" in cache`) + let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(account.name); + + if (cachedFound) { + const data = JSON.parse(cachedFound) + console.log(`found cache entry: ${data}`) + non_circulating_supply_ncheq += total_balance_ncheq(data); } + } console.log(`Non-circulating supply: ${non_circulating_supply_ncheq}`); // Get total supply @@ -55,270 +45,10 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P } export async function handler(request: Request): Promise { - let addresses_to_exclude: string[] = (await CIRCULATING_SUPPLY_WATCHLIST.list()).keys.map(k => k.name); + let addresses_to_exclude: string[] = (await CIRCULATING_SUPPLY_WATCHLIST.list()).keys.filter(k => !k.name.startsWith("grp_")); - // let circulating_supply = await get_circulating_supply(addresses_to_exclude); - let circulating_supply = await get_circulating_supply(getTempAddresses()); + console.log(`get_circulating_supply($addresses_to_exclude (len=${addresses_to_exclude.length})`) + let circulating_supply = await get_circulating_supply(addresses_to_exclude); return new Response(ncheq_to_cheq_fixed(circulating_supply)); } - -function getTempAddresses() { - let addresses_to_exclude = [ - "cheqd102mzdp87r6tjy0ttlgsxmldjex68k9tuxnqkwn", - "cheqd1074lwwsya2nvgfklg403aa9jtkgl7nml5zylg9", - "cheqd108q99w63he0uq2xa28gxf7hkpeqt8r5uqt9ghk", - "cheqd109c4836wy0aa9ft7uhvu6z5yxx55zh3yqjfle0", - "cheqd10e4zkalt6uqd3cwgtr8g4ychjjhrwq5n743j65", - "cheqd10lffy494en7teacp7ty9lgj5m9u333ps8cjns6", - "cheqd10nse74dkac20enn3ame36u73llc2tmnuehpj5f", - "cheqd10pps0c8msgsvjzrx2zge0tl68x93dzzfyszhqq", - "cheqd10tu4wnj67d8cnpwycllwur6qnltg87xfshndpf", - "cheqd10yut7vc3uvrasju4aa6tpwdxrmwdn6auxxeqft", - "cheqd128fra0sj9fgz9pmupc7dnrupgr38nm86wq3r34", - "cheqd12ht2skvkw4nnhq6fc48vlkugl0myryymaxmhhh", - "cheqd12lvv4cqdznu4q4gnwm3x4tjfqa4de49v49quq2", - "cheqd12tt8jn5fdlvedmxn4356rvn362g77tz4yv5za4", - "cheqd12vdx3selza7mk0qw6njtupfmkfk4nzl2jyqac7", - "cheqd12ys2yeqwa57083xhd5wjgjjp69f7jafqd6q4zr", - "cheqd130g8d4t7x5vpkdvuekzltghcv7re6ul35dgcq2", - "cheqd13duhx9h5g9z7qxy2u0uzselv2hljmzzslae5l5", - "cheqd13tgpmr4ueuw022l6nxkew823czawc00j56uh9w", - "cheqd13w44vy7s6mrqpxwc5ujpjfcldjzhkx5xfzeq94", - "cheqd13wvtjk3v5g4q7pema74nv9lrarr8e6vlx93zat", - "cheqd13x8zvv4a36mpcp278m53r5wesfsx2wwsrsn6kn", - "cheqd144j9sw5884vwnlem6sedtzj8rgsmaz7zsa6r96", - "cheqd14av5en2qpj3fgzyt2krx9kt3mlp8cn8mzqppfs", - "cheqd14c9fyc55szuwep6y2krnrqvm7qs97ret3uyjps", - "cheqd14gxh72xdtx5hh7ctg70lg3yy3gratlexsaj8ze", - "cheqd14hj6lrj3eaqlg0syf642eng3a2qm743kdaaddp", - "cheqd14re975jg2e3q2r60e0xdag77n8mz8t0mpffe9z", - "cheqd14ucjrgm8jm5z7yv322lgz3w9wz2psve7wncv5q", - "cheqd14vnqa3msv72ucp4j3lqhpvsmxuyu89qzvlazpk", - "cheqd14vyyr9a57mspj4kugg6yylcuqkxytm2q4srzra", - "cheqd14y0hce2rclpmcn8w0vvffc4f8fjlghduzn09pn", - "cheqd152wh3482jks8me22l085c7jcxtk74rm9yrhv08", - "cheqd157kn6s6p74ps7kmgnr6vvrqj47sx2895mxvr0f", - "cheqd15dg3zjxru3pus6qu5spn4g7n8d6f68wtslvssn", - "cheqd15erqx74e4q532cxp9jgsqc5gc045kaefnuakav", - "cheqd15eu3jgyezmct6yhanh3dpwsfn9qrulkna7q4rx", - "cheqd15gjydlputcmwxudnasc6c6sla2zy0cjs88svyd", - "cheqd16qcyenxplnsrgrpnnqmxgv2p2ddqwnjpz9quy3", - "cheqd16umet2xymlj8xtnqzr34vd06sml3fz8ujlm5en", - "cheqd16xz82c9u3w3yfsx6kjrrj39v5nx2lflysyf5ay", - "cheqd16zs74amn0es66vrryevr3pfa25pjwyllv575jv", - "cheqd173k83wpkgtr6xtx26caumsqkpwplpt0k0vr0y7", - "cheqd1774h8jzpv2kk5pasdrumknauypnm05nm6qjcsv", - "cheqd1782hcsf7m88pswfk2kn3zeujjzuykgzs5f00j8", - "cheqd1786x2uuudf5clle7s4e6y3safl4hwkhtgymdny", - "cheqd17c4jf69dl9d38aw5uysn8clf95xs8t74w33m6z", - "cheqd17jhrply2h0xk0zwku6859dr0y693379tunwcnl", - "cheqd17l9yrd0gt2dkgqdu4nknwdr7g7c22y4k94s2pl", - "cheqd17lqrgwg8lx8hp69jhku3t4hxx86qpw5cgx5l4n", - "cheqd17p5tyat3nr2kyqt49rak05s8ceftq6ykx5qt4t", - "cheqd17wrwqxsfwy4rqlltjhj6jxtz68tvxm0ykge5dr", - "cheqd17xekaxgwvn04dqmda37n5v5yatwe0pwm6umw2j", - "cheqd17xh3a5qvqy3tp4y9hp482ssl3pe2gq9j5m8zmh", - "cheqd17zjfjw5emz9r48wtma2hwhql334s0f375jvx72", - "cheqd182d4zh33fmgj38yhqv0ryapuancagvpj52frye", - "cheqd187k4vzggkx0kdurxg3xffdeehnfw4m0hm7h27s", - "cheqd18e9am44nt43w4qdg3s402ymzrvfdc37qd3pyt9", - "cheqd18jzpf9mvpaprtccvvsvpunv758p4avkk62nvd9", - "cheqd18n4a450pwne88u8g5p53pufdyaqhlxes6v5hwf", - "cheqd18p5t6pru4nnh8qmsnf3w90uqvk2e9aezlhym53", - "cheqd18pcq5jxm8vz0jkmm484y7n6g49ysclkz5jwr2e", - "cheqd18pwcnw73tsg3gguukwgx50u2d62zk4hspv8l32", - "cheqd19cjct09g4a50yqjavuflscqfqcqns2yp05n5ms", - "cheqd19tql8zv4kpp7umgx0nn6nt7qf0k29kx4h4trpe", - "cheqd19wxf36lwgkktuampg3w6uaw7grsy3gnrxj4t4h", - "cheqd1a25g2fln0racf9dlfn3g6e3vezvgynnez8uv0z", - "cheqd1aasqs6cn3hy2pqsypd6az0y8f5va32hhpxz8gh", - "cheqd1avu2ll646ypnjqa542rqpe0g26cfesve337ml8", - "cheqd1awfupskqfhev874vfjnpw9t0v94uuqhk74q3d4", - "cheqd1c3kwrlxsy3yz3phpap20z20j5qr2du3clgzs30", - "cheqd1c6g9nnlw2ht38ak3cpxu7p4yjaeef8m7frc0xp", - "cheqd1cgmzu9f9heas5ta46drxpeuk5yyn8v36yjlje7", - "cheqd1ck6rv3mst963m3pm5k8rg66elera8v26pah03f", - "cheqd1czauh9jmwgh9yc090gnhvmk337kz3jnarwuxed", - "cheqd1ddthz5uhx2382av5ew9t7yx25dznwhy0rpzzz3", - "cheqd1dksptzr757fe9tl23ywmp4guucsz0fl8e8ggll", - "cheqd1dp2tqng7lkm094t0g85etjet5rvlcue6hynx6p", - "cheqd1dqql2afmdc5wwzl74mu6f3jg5fxk6ksy6z7ma8", - "cheqd1dvxce4elk8lfph8c52mlevgkyg5lxdw6vgjf8h", - "cheqd1e4tdx3xfchd5vj0tmqgtdpkmlsftqvej8j9l6v", - "cheqd1e6y9pqk538qwdsra4y44a3qjll9tmgx44tx8e2", - "cheqd1e7jpqg3960msznsghctj9cxv4w7w4qph6g8jlz", - "cheqd1ecxmdrr5fhghskvawdyulqrczm604kx8dr5lgg", - "cheqd1ekt65hyudh5lhzjw55ctxqgtkddjeykfcqljkv", - "cheqd1eld0vlqy3tlmkhf7lgdcefc8qz7ny0cdk9j2ry", - "cheqd1ereqx3s4c3grc78w8ta2nmjk7a35p694rkt4zc", - "cheqd1f38cesgyn8yhkl9lhrxv4urr9gtl444k02uxac", - "cheqd1f9aunuufxy6euayzegxvp4jy3uy2xzcuarnmsl", - "cheqd1f9xhnvmdetn9mkh0w3pcll9rzy90uhwj5c0k9k", - "cheqd1fkxggf3m5ptap7h7fyh9ht095khuar026crx3c", - "cheqd1fqfwhraax5ngk0upwknej0jlpxc3ev7me7ht7s", - "cheqd1ftt2485zh3yn0t850atw34m2ctwvghsyf6kdr6", - "cheqd1fww87qsvqchyq9xwqcj5w550f2kxu6eyf75wsu", - "cheqd1fxkzl28rqsmjrl48qhkrzug42fhwg4xxtyuvgq", - "cheqd1gal47c74jets5ref6wjxguzqmkgry73z4u8tdh", - "cheqd1gp2nftpztxh44ex3vjqwfdvnvj2fkwtpn3g4gg", - "cheqd1gq6sn7fdeeplltyuv0tp76hq7766v64w8yktxj", - "cheqd1gzzlq30nwlctxdjg36fmy8djjycs2puvmmnmke", - "cheqd1h6hktd76fa93zt078nlrcrdndk7ynmg04an0fh", - "cheqd1h8mxrwh2n9gklz52m3wx0zr9230m727cjrsxw9", - "cheqd1hmc79es4m72l6yy4dmya4dclhnzdp3550ykfy8", - "cheqd1hmhktc89jmzhyf9y6qt7dzc7cy7cgu3cnqs6wp", - "cheqd1hn9m0pjxp4p4003xnzcmze7gydhg8a8r0858vd", - "cheqd1hprhwj8cny5p3w2nt9av3w0d9kr80ttdhagjte", - "cheqd1hzslf2k0a7k722vzvamyrt7tfq3447xk5ggfvg", - "cheqd1jcm85acd9jvvz6u43v3f2pk9lncatu879c4a8u", - "cheqd1jhttw93wzjrcjm8a74g8aph3wnpjfnqd9568t6", - "cheqd1jp00saqjfcdxkxqf9e8vrpyd4at9xlc7gf2syg", - "cheqd1juq5zgzjr92c4wn8cy39g3sws28du0crmfpjuc", - "cheqd1k86579yh9j49q9xtp4ta55zy6ad9azhrraal7k", - "cheqd1kj93ee7zkw44p9h0fnd6f8yhzeh509wg4acjjn", - "cheqd1kn3vj97l4k8t55073ragpk9zeufrnzh80ah9vs", - "cheqd1kputn5c3f0hn7da98lqf4c35fllxxeycrldutr", - "cheqd1kqx7q0dr3pn8hyxpu25gaz3m3qs835ldklmumk", - "cheqd1kr7dv9n6dnmjy0dldqxtp5dgq0jh6uhd5hx9z2", - "cheqd1kv5fha5ylprfd558f7r0uyex8rek8etpfxc6wu", - "cheqd1l8hvrw2j82er654s3dvvj9w9jjkcekqsx0jz8w", - "cheqd1l8uy4yn6utfgy65smmw40m8c4uyarw2hc8pkw9", - "cheqd1la5t260ggu3mkc7aha3ftwj40laynygy9chevy", - "cheqd1lg0vwuu888hu4arnt9egtqrm2662kcrtkqz49j", - "cheqd1lhch7spf87rgnev56xnqh2dqhpfk3eatdnth5y", - "cheqd1lqtfptxeurmd54xy7zkns7qwwutvns0w4ufyft", - "cheqd1m6uas09cczhyra3up3tn4calwvl7ygxvxym7a0", - "cheqd1m7d75z29hqu2d0uv0mk2s88um5hvj259md7758", - "cheqd1ma754mlx2rdvshae9t8jkz2c4x7ft2lpkmn8vt", - "cheqd1mjlhlt3yx9tqjsnprtsxemf9hwxwxgd3r6xrnr", - "cheqd1mw6xpmqt7u7mlcu0lerh7dw5c9rvsqsthqcgrw", - "cheqd1nfsz8r4p359fsctk0lt4ee3kma0zg89xldxpqu", - "cheqd1nqzypqthx348llfuj9j5xftzfua2l75qy9fwq6", - "cheqd1nu5aaszuxjvxt5x8eq2s8tlad4nwjk2acn8cj3", - "cheqd1p02e0x204a49utmkgwjdk9kpy8gqtcm7mpu87l", - "cheqd1p9szkzftfddxce82ymckgxuxwtqc6d4k2at4ep", - "cheqd1pc02hdy79wuwyxzy7ucumnmaexp9gx5wa5ckqr", - "cheqd1pdu0r224x6tvfkhswnqrq9z5w4d73jns4u3k2m", - "cheqd1phmz70wgzyahlacn78hwzdkwu5fky5tfl8gfs3", - "cheqd1pqqlxl3msedxcej57q9examfzhu0ttzdsp02pw", - "cheqd1psrm6t3hqwzvmfnw6x4ysf0hd240a3k3d9yy27", - "cheqd1py8mmqqv3wk22ae4pz0r0da7x9zzrxefzugklh", - "cheqd1pydlf8h7ezlkpwq8ud3gdx83ra9me47z48qtat", - "cheqd1q3k2ne32vtx2dvf8ee7vpafcvdsrsvqx5thf72", - "cheqd1q449p4s8c4t8chdrx9gk5nlqt43v508erl7qsu", - "cheqd1q5cazs6yuvmcf34z369z2fvt2ec6n2f7vp06hm", - "cheqd1q8ghndxp657hajzsk4trfkuh5zrm74vedyhplh", - "cheqd1q9tznprszw5usu55yfk42fenfe6tyq58j53skg", - "cheqd1qaa9pd60g6ry9u3kf9sezw26lyejky8xayr52s", - "cheqd1qc7nexzm8egzte8amh0qusg99xjsnc2w7m96c5", - "cheqd1qn78q6tddxxz2rpcflsuququvuvjalrunxp3c7", - "cheqd1qs0nhyk868c246defezhz5eymlt0dmajna2csg", - "cheqd1qvj2uuyvalx22lqnehh3awjcq35a0rz5gpudtm", - "cheqd1r2g7e6ymdwfql5j87gs0xv5llh9xp692vfwruq", - "cheqd1s3hqzaraajvlkpl4fseteenad2zev9hmhfdhrd", - "cheqd1s7qgtdpyw9cj2swkdhzywwlssxapndcalrfy5q", - "cheqd1s8f3rqprpwhn0vxl6me76qwg6y9cpxj2yjjy3l", - "cheqd1s9wn3wxea4maef8t04jyusnpc29p58v83eesz0", - "cheqd1scxe3jqyuqv0flgl0w3s95hhy9jsp98h5a24pk", - "cheqd1sr0066h5p2ncfc8hnrgqlw3lqxgylwn06ccl9e", - "cheqd1sv4w8tadeulklgwpu477ls3rvd2se3cuftdvkr", - "cheqd1t0qhk7kqe6vv36alehrwn2ht5e0dc8wh2uzkr6", - "cheqd1t0vw4kha8ffw0sfsd2k56287q73zwnca5gr7l2", - "cheqd1t3gsdl2uux0rmt3acn8qursc9sarhtm6mpa3hq", - "cheqd1t5pl7x25egk2xcfu2kyg4a7ga4hyg2g5anv5hh", - "cheqd1tgtgmksy2de42nzahk9kwml9crf36r735xwazj", - "cheqd1thhqk2q4s8h3ugwjm4h2v09gxaturfzkyjeux3", - "cheqd1tml0chnp3mee88cyk63la3ctj9ndu4vtl2e2m7", - "cheqd1tpvhkuvn8je2qdghexfsgeg0q37zcqej06m30x", - "cheqd1tqpxjdnxcjqswrcq5slz64wrh7t5j6jle7gwfh", - "cheqd1tt5p2xwz3xv2d4wjkegycu6pauv3jxyta3vj0r", - "cheqd1tx65lajxvzwmvfxwvev23kxac2v3zj3jg0hrcw", - "cheqd1u3uwj2rvxt6ytweypemcc35vy80zwlnx7ly6wt", - "cheqd1ucxsrv5yyvxhjk2cu3gl54l94dru7apgwrlk9q", - "cheqd1uvjqw3wadfjycznpe0z8u36aqnecndtzlwgf7s", - "cheqd1va3slq5k97a008gxncn9eavvf4mt84xcmej65l", - "cheqd1vex9phvwcwr785k2jal7tv2pklysqx94kzay8a", - "cheqd1vh0rar0jgyx8xeett26nv77jnvxr99k8q8qa6q", - "cheqd1vj9qtf9fy00q8xnwxfwsswtr24ajl8a592hyp8", - "cheqd1vl23k3x73wzq4t7x3sh54n5f40kpse32pleka6", - "cheqd1vlu2azc2fs9p5zffyeefth0l79eygwj8vmt2r6", - "cheqd1vusv4s233mw867ua63m7upjxzgcyqhwhumqe6d", - "cheqd1vxfyppcnu6lz72mdc706mlv7kx8434jaknhgt6", - "cheqd1w5xsmev2zv59yhtk5nsl6xrdwpu588k5su6xs0", - "cheqd1wsf8ns8gmcq76nes484pt46c9ru9c9mdmwm7sc", - "cheqd1wwwncrnde9myewpn8j00xeyt4hfs3sqmkyasqm", - "cheqd1wznaxsqya0meesp9f8xrknnreuszfd8kup3mq5", - "cheqd1wzyfq9ujg7vp3whn7nh7w5ner8p7ghk2pxw0g5", - "cheqd1x7ku9c2x2j263yaj48ct9nwjark75f0hhgmpvg", - "cheqd1x9dc5enk7f77apgscc3emep5nz0fapxm4w9r9m", - "cheqd1xqgm3nw0uh4k4zj0u4l2ljfqslrfumff5syu49", - "cheqd1xqxz5wneg2hqg9tugzuv8v8wgyr2pl32v2h3hl", - "cheqd1xs7kvk7n7m6ca6yw7ysxxfulwauc2s46z2w283", - "cheqd1xusdhhtlqk0vetgqg7nu39j9q62zq63q22k463", - "cheqd1xwnu2mm96uexuergqk0kqt4ds84x9zsjyne49e", - "cheqd1xyyygaxx384k07sv3rk9zj7dezuzf250pahr0w", - "cheqd1y6ekzultqshf8xwtv46mlzfquj6ys3vmncascq", - "cheqd1y8edvwl536rxuqdznh48mh6mg648gf9a3xx2s9", - "cheqd1ygwl8m54cpenq7t5v5p0y9xau3gmg873fqlqag", - "cheqd1ylfukxalf0zr2wdma0gx33e9fccha94qr59rj4", - "cheqd1yp52vx7sjv2dhhfqe2n52ntr0av39lye9fx5gl", - "cheqd1yqj0s0htaseqc9prkcsmucme3ef5fsyq5mkqrd", - "cheqd1zh8g8lw0zhp0qk0fxjq9n5p2yjc4xgpca5gzds", - "cheqd1zhqu2pzpsv0x5afmw3tm5579pqxy59msh8ytj3", - "cheqd1zhw4qtrd04kp7zz329zjlwy0s3vjunxua6rrqd", - "cheqd1zm2pz9gj4vsl278vydapx0x8e3fguvxalg7n98", - "cheqd1zmxl82zfgqsc0efuyhateaen8mzp7ngkaw4qxz", - "cheqd1zu4mm0lzhm7az4fwc6d28pkt2vvpte5ye45rgv", - "cheqd1zym5ync279m2sxx99dhtv4s00apr08lzw2g8g8", - "cheqd1zz0gck564g3dvaxctccc5kgxw2sgtes7p2led6", - "delayed:cheqd10vmxqqnc9j7axh808ymyz8t8kneux5n8xmmhq9", - "delayed:cheqd14c5zvsf74uayyp90mav49gx2aj250s8u8yv7y9", - "delayed:cheqd15yd3fnn3ev5xeacazejgejd5jxrnpusg8mgu8h", - "delayed:cheqd15zw9j5e3l4l3k6fex8e7hznzmf0mpz5vcwlrt3", - "delayed:cheqd17n6mefp8vhyrtvavc9fd9hg0hx624skgdjtrpz", - "delayed:cheqd184z8z58dmax77csg0a9tneku0sqskeqymx5hxw", - "delayed:cheqd18jc49zfkew8aj8zmxjyrk3nr8asqu8jxg56fwz", - "delayed:cheqd198yncktukks0aqtyk4vq0hw20v67ffr0rm8cla", - "delayed:cheqd1a0vumnau34trnvjpnqr4ywwaqd89l4s7ytwkk5", - "delayed:cheqd1ajefdpfng3l66gr2j3ygvh6efrr6twrd3968rk", - "delayed:cheqd1c5wva37ufnpdypxkxeya9kawrzgay6evnra50u", - "delayed:cheqd1cpj9vd5e3dyvyj0nds3hanz30zafakdc4h00wj", - "delayed:cheqd1crws5sz8tdl37v40tjn0lmlu6ztjhxar9jklqz", - "delayed:cheqd1dy69x2g8vj0sdnutr29lt2wqgxqw98pcxltuu5", - "delayed:cheqd1eraud0nswfglksl9tl8e5dlemrq5eualjftv3z", - "delayed:cheqd1ewhshdq9cdhxwx8vrnnffhfe8qw3szryqv4gu7", - "delayed:cheqd1exe54lymn5mhpjs0vj6yppnecdmtfss2qcd4cq", - "delayed:cheqd1g3379fkvfc6jgxa66dswzsmzeguryv6zkxca87", - "delayed:cheqd1gz3suxvpqrdwntnn7r95eerwa3ura92kp4tm8c", - "delayed:cheqd1jkzd7sn6dzn54tl0y4ztw7n68r0p4aw53f5mqx", - "delayed:cheqd1k3vc8xplwtqpyt55ysx085lhr09xxvhuq9ddra", - "delayed:cheqd1kaascflyltj3pp54hwarnat6kmwtw2zvllsqdc", - "delayed:cheqd1kq5atufq4ryhdld8s7cmy2735uxhh6wdakj9cj", - "delayed:cheqd1kysvc7hq8944jq68yjg7hqxud66r3ajx8uv5s8", - "delayed:cheqd1la5de5x52qw8vspwj70g6c6mem02al8sf4z4sm", - "delayed:cheqd1lp0lq9mhc22y42znga5mvjngjpdj4pgvgzs950", - "delayed:cheqd1lvgq858vk9hs9stfkkv0dhhvnwv5eczjx4rkjw", - "delayed:cheqd1m5r4emnpmgnl0xgptk4f4pkrqmnlrzpnhs8aqu", - "delayed:cheqd1mn83wm6f43f9ls6lsju8w2x4z4rg4rcvmcsq7z", - "delayed:cheqd1nn7rcyvfd3zqhumuwwj05wvdh4x28f2qs3sex5", - "delayed:cheqd1ppuya3ezvs2jq7n2e0ygec6r3uhx0gea6hl9vp", - "delayed:cheqd1pw0z58t0jrtexp26694ycrdgv6ycmt5fcy00vg", - "delayed:cheqd1q4lkgp3psk2fdxt2pgr0lujyj5fk4lz2wlz8dy", - "delayed:cheqd1q6wwcqf902ygu8z7dcyhh8ca0u6a5g4kns6he8", - "delayed:cheqd1qxr4jdgug8dve9kq6vc6pwdhpuyvlsql85xx0z", - "delayed:cheqd1r2y0ffh5xgkrnzelygz35mprjejaatnah9untf", - "delayed:cheqd1s0sngupwnkfv84r079x5ct74j8nnxgjfp89p54", - "delayed:cheqd1ssge5ykw4hvu4jy6eeen7hyxnw72mds6mzj79h", - "delayed:cheqd1ssjkeq5k385fydn9n7qvcp74pqkxgl8r8eczfq", - "delayed:cheqd1sueav9pmjy5k8crpg2nuakyssd86kcrrlc0qre", - "delayed:cheqd1t62uzdkyaw27v59ufyftzccl707k8a6rt9kcgn", - "delayed:cheqd1w53yz6ylpraykng3sw6r8lunuar7lteavel3x0", - "delayed:cheqd1x6srspstwx3pdta64d99en9radmaztms6lj0te", - "delayed:cheqd1y3fvu2ychq7247fkwfna0nn6exl9w6u009tnh7", - "delayed:cheqd1z8hvr2drwgnfqudkrqhwvdv5p40h8maw7jg92g", - ]; - - - return addresses_to_exclude; -} diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index 5f02eb45..5eceab3e 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -11,29 +11,24 @@ export async function updateAllBalances(group: number, event: Event) { console.log(`found ${cached.keys.length} cached accounts`) for (const key of cached.keys) { - let addr: string; - if (key.name.startsWith("grp_")) { - const parts = key.name.split(':') - addr = parts[1] - } else { - addr = key.name - } - const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key) if (found) { - const item = JSON.parse(found) - // only update errored for now - if (item.balances.error) { - const res = await updateBalance(node_api, addr) - - if (res !== undefined) { - const data = await res.json() as Account; - console.log(`updating account (grp_${group}:${addr}) balance (${data})`) - balances.push({ account: addr, balances: data }) - } + let addr: string; + if (key.name.startsWith("grp_")) { + const parts = key.name.split(':') + addr = parts[1] + } else { + addr = key.name } + const res = await updateBalance(node_api, addr) + + if (res !== undefined) { + const data = await res.json() as Account; + console.log(`updating account (grp_${group}:${addr}) balance (${data})`) + balances.push({ account: addr, balances: data }) + } } } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index c35886bb..48805561 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -8,7 +8,7 @@ export async function updateBalance(node_api: NodeApi, address: string): Promise const account = await bd_api.get_account(address); try { - const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1.${address}`) + const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1:${address}`) if (cachedAccount) { console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) diff --git a/src/helpers/node.ts b/src/helpers/node.ts index ebd459b5..f58653b4 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,26 +1,25 @@ import { Account } from "../types/bigDipper"; import { Coin } from "../types/node"; -import { ncheq_to_cheqd } from "./currency"; export function total_balance_ncheq(account: Account): number { let balance = Number(account?.accountBalance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); let delegations = 0; if (account?.delegationBalance?.coins && account?.delegationBalance?.coins.length > 0) { - delegations = ncheq_to_cheqd(account?.delegationBalance?.coins[0].amount); + delegations = Number(account?.delegationBalance?.coins[0].amount); } let unbonding = 0; if (account?.unbondingBalance?.coins && account?.unbondingBalance?.coins.length > 0) { - unbonding = ncheq_to_cheqd(account?.unbondingBalance?.coins[0]?.amount); + unbonding = Number(account?.unbondingBalance?.coins[0]?.amount); } let rewards = 0; if (account?.rewardBalance?.coins && account?.rewardBalance?.coins.length > 0) { - rewards = ncheq_to_cheqd(account?.rewardBalance?.coins[0]?.amount); + rewards = Number(account?.rewardBalance?.coins[0]?.amount); } - return Number(balance + delegations + unbonding + rewards); + return balance + delegations + unbonding + rewards; } export function delayed_balance_ncheq(balance: Coin[]): number { diff --git a/src/index.ts b/src/index.ts index ede1d923..821dc984 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,9 +12,11 @@ import { handler as balanceUpdaterHandler } from "./handlers/balanceUpdater"; import { updateAllBalances } from "./handlers/cron"; addEventListener('scheduled', (event: any) => { + console.log(`triggering scheduled account balance update`) + event.waitUntil(updateAllBalances(1, event)); - event.waitUntil(updateAllBalances(2, event)); - event.waitUntil(updateAllBalances(3, event)); + // event.waitUntil(updateAllBalances(2, event)); + // event.waitUntil(updateAllBalances(3, event)); }) addEventListener('fetch', (event: FetchEvent) => { diff --git a/wrangler.toml b/wrangler.toml index d550c5bc..239d9a46 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -88,7 +88,7 @@ binding = "CIRCULATING_SUPPLY_WATCHLIST" id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" preview_id = "1e1032cbf6854d88b12317da8a792928" -#crons = ["0 * * * *"] +crons = ["0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From f5a5bf3c07c78a4d417acf8738e84a6d4086f851 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Wed, 28 Sep 2022 07:07:16 -0300 Subject: [PATCH 026/224] Removed single account balance updater temp endpoint --- src/index.ts | 3 +-- wrangler.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/index.ts b/src/index.ts index 821dc984..880e8a58 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,7 +13,7 @@ import { updateAllBalances } from "./handlers/cron"; addEventListener('scheduled', (event: any) => { console.log(`triggering scheduled account balance update`) - + event.waitUntil(updateAllBalances(1, event)); // event.waitUntil(updateAllBalances(2, event)); // event.waitUntil(updateAllBalances(3, event)); @@ -38,7 +38,6 @@ function registerRoutes(router: Router) { router.get('/supply/total', totalSupplyHandler); router.get('/', totalSupplyHandler); router.get('/_/grp/:grp', balanceUpdaterHandler); - router.get('/_/:address', balanceUpdaterHandler); // 404 for all other requests router.all('*', () => new Response('Not Found.', { status: 404 })) diff --git a/wrangler.toml b/wrangler.toml index 239d9a46..3b639ab0 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -88,7 +88,7 @@ binding = "CIRCULATING_SUPPLY_WATCHLIST" id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" preview_id = "1e1032cbf6854d88b12317da8a792928" -crons = ["0 * * * *"] +crons = ["0/60 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 9027ce66eba45c9bdca128c811d20af55ee83b98 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Wed, 28 Sep 2022 07:12:21 -0300 Subject: [PATCH 027/224] Improved error logging for graphgl (workers requires a Map for logging objs) --- src/handlers/circulatingSupply.ts | 2 +- src/handlers/cron.ts | 2 +- src/helpers/balance.ts | 13 +++++++------ src/helpers/graphql.ts | 8 ++++---- src/index.ts | 4 ++-- wrangler.toml | 4 +++- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 12323f10..d9cb5c02 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -25,7 +25,7 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P if (cachedFound) { const data = JSON.parse(cachedFound) - console.log(`found cache entry: ${data}`) + console.log(`found cache entry: ${cachedFound}`) non_circulating_supply_ncheq += total_balance_ncheq(data); } } diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index 5eceab3e..e89e8d25 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -11,7 +11,7 @@ export async function updateAllBalances(group: number, event: Event) { console.log(`found ${cached.keys.length} cached accounts`) for (const key of cached.keys) { - const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key) + const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name) if (found) { let addr: string; diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 48805561..0701109b 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -2,21 +2,22 @@ import { NodeApi } from "../api/nodeApi"; import { GraphQLClient } from "./graphql"; import { BigDipperApi } from "../api/bigDipperApi"; -export async function updateBalance(node_api: NodeApi, address: string): Promise { +export async function updateBalance(node_api: NodeApi, addr: string): Promise { const gql_client = new GraphQLClient(GRAPHQL_API); const bd_api = new BigDipperApi(gql_client); - const account = await bd_api.get_account(address); + const account = await bd_api.get_account(addr); try { - const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1:${address}`) + const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1:${addr}`) if (cachedAccount) { - console.log(`account "${address}" found in cache: ${JSON.stringify(cachedAccount)}`) + console.log(`account "${addr}" found in cache: ${JSON.stringify(cachedAccount)}`) if (typeof account === "object" && account.accountBalance) { const grpN = Math.floor(Math.random() * 3) + 1 - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${address}`, JSON.stringify(account)) - console.log(`account "${address}" balance updated. (res=${JSON.stringify(account)})`) + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, JSON.stringify(account)) + + console.log(`account "${addr}" balance updated. (res=${JSON.stringify(account)})`) return new Response(JSON.stringify(account)); } else { diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index 9b87cf51..e8e68d10 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -14,16 +14,16 @@ export class GraphQLClient { body: JSON.stringify(req) }) - let json = await resp.json() + let json: { errors: any } = await resp.json() if (json.errors) { - console.error(json.errors) + console.error(new Map(json.errors)) return {} as T; } return json as T; - } catch (e) { - console.error(JSON.stringify(e)) + } catch (e: any) { + console.error(new Map(e)) return {} as T; } } diff --git a/src/index.ts b/src/index.ts index 880e8a58..29c4c308 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,8 +15,8 @@ addEventListener('scheduled', (event: any) => { console.log(`triggering scheduled account balance update`) event.waitUntil(updateAllBalances(1, event)); - // event.waitUntil(updateAllBalances(2, event)); - // event.waitUntil(updateAllBalances(3, event)); + event.waitUntil(updateAllBalances(2, event)); + event.waitUntil(updateAllBalances(3, event)); }) addEventListener('fetch', (event: FetchEvent) => { diff --git a/wrangler.toml b/wrangler.toml index 3b639ab0..c26ec2f5 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -88,8 +88,10 @@ binding = "CIRCULATING_SUPPLY_WATCHLIST" id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" preview_id = "1e1032cbf6854d88b12317da8a792928" -crons = ["0/60 * * * *"] + +[triggers] +crons = ["0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### ############################################################### From 97d6523a47b69ceb6ad1b35b5fdeaa69110215ae Mon Sep 17 00:00:00 2001 From: drgomesp Date: Wed, 28 Sep 2022 10:11:56 -0300 Subject: [PATCH 028/224] Removing duplicates from KV while procecssing --- src/handlers/balanceUpdater.ts | 20 +++++--------------- src/handlers/cron.ts | 20 +++++++++++++++++++- src/helpers/balance.ts | 13 +++++-------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index 71ba8133..8a6779d7 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -1,24 +1,14 @@ -import { NodeApi } from "../api/nodeApi"; import { Request } from "itty-router"; -import { validate_cheqd_address } from "../helpers/validate"; -import { updateBalance } from "../helpers/balance"; import { updateAllBalances } from "./cron"; export async function handler(request: Request): Promise { - let node_api = new NodeApi(REST_API); - const address = request.params?.['address']; + const grp = request.params?.['grp']; - if (!address || !validate_cheqd_address(address)) { - const grp = request.params?.['grp']; - - console.log(`updating all account balances (group: ${grp})`) - const res = await updateAllBalances(Number(grp), {} as Event) - if (res !== undefined) { - return res - } + console.log(`updating all account balances (group: ${grp})`) + const res = await updateAllBalances(Number(grp), {} as Event) + if (res !== undefined) { + return res } - - return updateBalance(node_api, address ?? ""); } diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index e89e8d25..6b8c7872 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -18,15 +18,33 @@ export async function updateAllBalances(group: number, event: Event) { if (key.name.startsWith("grp_")) { const parts = key.name.split(':') addr = parts[1] + + switch (parts[0]) { + case 'grp_1': + await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_2:${addr}`); + await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_3:${addr}`); + break + + case 'grp_2': + await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_1:${addr}`); + await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_3:${addr}`); + break + + case 'grp_3': + await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_1:${addr}`); + await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_2:${addr}`); + break + } } else { addr = key.name } - const res = await updateBalance(node_api, addr) + const res = await updateBalance(node_api, addr, group) if (res !== undefined) { const data = await res.json() as Account; console.log(`updating account (grp_${group}:${addr}) balance (${data})`) + balances.push({ account: addr, balances: data }) } } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 0701109b..cb716598 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -2,30 +2,27 @@ import { NodeApi } from "../api/nodeApi"; import { GraphQLClient } from "./graphql"; import { BigDipperApi } from "../api/bigDipperApi"; -export async function updateBalance(node_api: NodeApi, addr: string): Promise { +export async function updateBalance(node_api: NodeApi, addr: string, grpN: number): Promise { const gql_client = new GraphQLClient(GRAPHQL_API); const bd_api = new BigDipperApi(gql_client); const account = await bd_api.get_account(addr); try { - const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_1:${addr}`) + const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`) if (cachedAccount) { console.log(`account "${addr}" found in cache: ${JSON.stringify(cachedAccount)}`) if (typeof account === "object" && account.accountBalance) { - const grpN = Math.floor(Math.random() * 3) + 1 + // const grpN = Math.floor(Math.random() * 3) + 1 await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, JSON.stringify(account)) console.log(`account "${addr}" balance updated. (res=${JSON.stringify(account)})`) return new Response(JSON.stringify(account)); - } else { - return new Response(JSON.stringify({ error: new Error("something went wrong") })); } } - } catch (e) { - console.error(e) - return new Response(JSON.stringify({ error: e })) + } catch (e: any) { + return new Response(JSON.stringify({ error: new Map(e) })) } } From c4155576352bb3f4e044146db683542601d87757 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Wed, 28 Sep 2022 11:18:59 -0300 Subject: [PATCH 029/224] Removed temporary account removal from KV --- src/handlers/circulatingSupply.ts | 7 +++---- src/handlers/cron.ts | 17 ----------------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index d9cb5c02..56f550f6 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -37,11 +37,10 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P // Calculate circulating supply return total_supply_ncheq - non_circulating_supply_ncheq; - } catch (e) { - console.error(e) + } catch (e: any) { + console.error(new Map(e)) + return total_supply_ncheq } - - return total_supply_ncheq } export async function handler(request: Request): Promise { diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index 6b8c7872..562c7b07 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -18,23 +18,6 @@ export async function updateAllBalances(group: number, event: Event) { if (key.name.startsWith("grp_")) { const parts = key.name.split(':') addr = parts[1] - - switch (parts[0]) { - case 'grp_1': - await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_2:${addr}`); - await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_3:${addr}`); - break - - case 'grp_2': - await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_1:${addr}`); - await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_3:${addr}`); - break - - case 'grp_3': - await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_1:${addr}`); - await CIRCULATING_SUPPLY_WATCHLIST.delete(`grp_2:${addr}`); - break - } } else { addr = key.name } From d36bc4fed57ca3df2cf3f4fbae1bf2ddef68cb24 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Wed, 28 Sep 2022 11:31:18 -0300 Subject: [PATCH 030/224] Better logging --- src/handlers/cron.ts | 2 +- src/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index 562c7b07..ae1001e9 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -26,7 +26,7 @@ export async function updateAllBalances(group: number, event: Event) { if (res !== undefined) { const data = await res.json() as Account; - console.log(`updating account (grp_${group}:${addr}) balance (${data})`) + console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(data)})`) balances.push({ account: addr, balances: data }) } diff --git a/src/index.ts b/src/index.ts index 29c4c308..416a5972 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,6 +39,7 @@ function registerRoutes(router: Router) { router.get('/', totalSupplyHandler); router.get('/_/grp/:grp', balanceUpdaterHandler); + // 404 for all other requests router.all('*', () => new Response('Not Found.', { status: 404 })) } From 272d2cd46296eb3732287cb767f7925552fa4a61 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Wed, 28 Sep 2022 11:45:25 -0300 Subject: [PATCH 031/224] Fixed endpoint --- src/handlers/circulatingSupply.ts | 3 ++- src/index.ts | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 56f550f6..5ba1af70 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -15,6 +15,7 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: 'grp_' }) + console.log(`found ${cached.keys.length} cached items`) @@ -46,8 +47,8 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P export async function handler(request: Request): Promise { let addresses_to_exclude: string[] = (await CIRCULATING_SUPPLY_WATCHLIST.list()).keys.filter(k => !k.name.startsWith("grp_")); - console.log(`get_circulating_supply($addresses_to_exclude (len=${addresses_to_exclude.length})`) let circulating_supply = await get_circulating_supply(addresses_to_exclude); + console.log(`circulating_supply=${circulating_supply}`) return new Response(ncheq_to_cheq_fixed(circulating_supply)); } diff --git a/src/index.ts b/src/index.ts index 416a5972..a7f1e58e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,6 +26,7 @@ addEventListener('fetch', (event: FetchEvent) => { }) function registerRoutes(router: Router) { + router.get('/_/grp/:grp', balanceUpdaterHandler); router.get('/', totalSupplyHandler); router.get('/balances/liquid/:address', liquidBalanceHandler); router.get('/balances/total/:address', totalBalanceHandler); @@ -36,8 +37,6 @@ function registerRoutes(router: Router) { router.get('/supply/circulating', circulatingSupplyHandler); router.get('/supply/staked', totalStakedCoinsHandler); router.get('/supply/total', totalSupplyHandler); - router.get('/', totalSupplyHandler); - router.get('/_/grp/:grp', balanceUpdaterHandler); // 404 for all other requests From 7fd96ca756b4c3cc74a8659cca57f8114f3e323d Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Wed, 28 Sep 2022 11:45:25 -0300 Subject: [PATCH 032/224] Fixed endpoint --- src/handlers/balanceUpdater.ts | 6 ++++-- src/handlers/circulatingSupply.ts | 10 ++++++---- src/handlers/cron.ts | 6 ++++-- src/helpers/balance.ts | 24 ++++++++++++++---------- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index 8a6779d7..da6f9733 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -9,6 +9,8 @@ export async function handler(request: Request): Promise { if (res !== undefined) { return res } -} - + return new Response(JSON.stringify({ + error: new Error("this endpoint should be called for all accounts") + })) +} diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 5ba1af70..3ca480e3 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -15,7 +15,6 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: 'grp_' }) - console.log(`found ${cached.keys.length} cached items`) @@ -24,10 +23,13 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P console.log(`looking for account: "${JSON.stringify(account.name)}" in cache`) let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(account.name); - if (cachedFound) { + if (cachedFound !== undefined) { const data = JSON.parse(cachedFound) - console.log(`found cache entry: ${cachedFound}`) - non_circulating_supply_ncheq += total_balance_ncheq(data); + + if (typeof data === "object") { + console.log(`found cache entry: ${cachedFound}`) + non_circulating_supply_ncheq += total_balance_ncheq(data); + } } } diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index ae1001e9..bc695bac 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -26,9 +26,11 @@ export async function updateAllBalances(group: number, event: Event) { if (res !== undefined) { const data = await res.json() as Account; - console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(data)})`) - balances.push({ account: addr, balances: data }) + if (!data.error) { + console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(data)})`) + balances.push({ account: addr, balances: data }) + } } } } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index cb716598..3050d458 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -7,22 +7,26 @@ export async function updateBalance(node_api: NodeApi, addr: string, grpN: numbe const bd_api = new BigDipperApi(gql_client); const account = await bd_api.get_account(addr); + if (!account) { + return new Response(JSON.stringify({ error: "Account not found" })) + } + try { const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`) - if (cachedAccount) { - console.log(`account "${addr}" found in cache: ${JSON.stringify(cachedAccount)}`) + if (cachedAccount !== undefined) { + console.log(`account "${addr}" found in cache: ${cachedAccount}`) - if (typeof account === "object" && account.accountBalance) { - // const grpN = Math.floor(Math.random() * 3) + 1 - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, JSON.stringify(account)) + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, JSON.stringify(account)) + console.log(`account "${addr}" balance updated. (res=${JSON.stringify(account)})`) - console.log(`account "${addr}" balance updated. (res=${JSON.stringify(account)})`) - - return new Response(JSON.stringify(account)); - } + return new Response(JSON.stringify(account)); } + + return new Response(JSON.stringify({ error: "Account not found" })); } catch (e: any) { - return new Response(JSON.stringify({ error: new Map(e) })) + console.error(e) + console.log(new Map(e)) + return new Response(JSON.stringify({ error: e })) } } From 77958ad5c347bd196a56840a489c3b4614b7f8de Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 30 Sep 2022 06:15:32 -0300 Subject: [PATCH 033/224] Added proper list of addresses to import (temp) --- src/handlers/balanceUpdater.ts | 4 ++-- src/handlers/cron.ts | 3 ++- src/index.ts | 10 ++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index da6f9733..d71c50fc 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -1,11 +1,11 @@ import { Request } from "itty-router"; -import { updateAllBalances } from "./cron"; +import { updateGroupBalances } from "./cron"; export async function handler(request: Request): Promise { const grp = request.params?.['grp']; console.log(`updating all account balances (group: ${grp})`) - const res = await updateAllBalances(Number(grp), {} as Event) + const res = await updateGroupBalances(Number(grp), {} as Event) if (res !== undefined) { return res } diff --git a/src/handlers/cron.ts b/src/handlers/cron.ts index bc695bac..fecf5f07 100644 --- a/src/handlers/cron.ts +++ b/src/handlers/cron.ts @@ -2,7 +2,7 @@ import { NodeApi } from "../api/nodeApi"; import { updateBalance } from "../helpers/balance"; import { Account } from "../types/bigDipper"; -export async function updateAllBalances(group: number, event: Event) { +export async function updateGroupBalances(group: number, event: Event) { let node_api = new NodeApi(REST_API); let balances: { account: String, balances: Account } [] = []; @@ -38,5 +38,6 @@ export async function updateAllBalances(group: number, event: Event) { return new Response(JSON.stringify(balances)); } catch (e) { console.error(e) + throw e; } } diff --git a/src/index.ts b/src/index.ts index a7f1e58e..1fdc0125 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,14 +9,15 @@ import { handler as delegatorCountHandler } from './handlers/delegatorCount'; import { handler as totalDelegatorsHandler } from './handlers/totalDelegators'; import { handler as totalStakedCoinsHandler } from "./handlers/totalStakedCoins"; import { handler as balanceUpdaterHandler } from "./handlers/balanceUpdater"; -import { updateAllBalances } from "./handlers/cron"; +import { updateGroupBalances } from "./handlers/cron"; addEventListener('scheduled', (event: any) => { console.log(`triggering scheduled account balance update`) - event.waitUntil(updateAllBalances(1, event)); - event.waitUntil(updateAllBalances(2, event)); - event.waitUntil(updateAllBalances(3, event)); + event.waitUntil(updateGroupBalances(1, event)); + event.waitUntil(updateGroupBalances(2, event)); + event.waitUntil(updateGroupBalances(3, event)); + event.waitUntil(updateGroupBalances(4, event)); }) addEventListener('fetch', (event: FetchEvent) => { @@ -26,6 +27,7 @@ addEventListener('fetch', (event: FetchEvent) => { }) function registerRoutes(router: Router) { + router.get('/_/grp', balanceUpdaterHandler); router.get('/_/grp/:grp', balanceUpdaterHandler); router.get('/', totalSupplyHandler); router.get('/balances/liquid/:address', liquidBalanceHandler); From 45987417fe7329a9307441ebe6149176a84cb915 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 30 Sep 2022 06:35:12 -0300 Subject: [PATCH 034/224] Implementing requested changes from PR review --- src/api/bigDipperApi.ts | 13 ++++++++++--- src/helpers/graphql.ts | 24 +++++++++--------------- src/index.ts | 1 - 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 86caeb0a..4c00c400 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -6,7 +6,7 @@ export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) { } - async get_account(address: string): Promise { + async get_account(address: string): Promise { let query = `query Account($address: String!, $where: vesting_account_bool_exp) { accountBalance: action_account_balance(address: $address) { coins @@ -41,9 +41,16 @@ export class BigDipperApi { } } - let resp = await this.graphql_client.query(query, params); + try { + let resp = await this.graphql_client.query<{ + data: any, errors: any + }>(query, params); - return resp.data as Account + return resp.data as Account + } catch (e: any) { + console.error(new Map(e)) + return null; + } } async get_total_supply(): Promise { diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index e8e68d10..a7e6724b 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -8,23 +8,17 @@ export class GraphQLClient { variables, } - try { - let resp = await fetch(this.base_url, { - method: "POST", - body: JSON.stringify(req) - }) + let resp = await fetch(this.base_url, { + method: "POST", + body: JSON.stringify(req) + }) - let json: { errors: any } = await resp.json() + let json: { errors: any } = await resp.json() - if (json.errors) { - console.error(new Map(json.errors)) - return {} as T; - } - - return json as T; - } catch (e: any) { - console.error(new Map(e)) - return {} as T; + if (json.errors) { + throw new Error(`query failed: ${json.errors}`) } + + return json as T; } } diff --git a/src/index.ts b/src/index.ts index 1fdc0125..abe09e3a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,7 +27,6 @@ addEventListener('fetch', (event: FetchEvent) => { }) function registerRoutes(router: Router) { - router.get('/_/grp', balanceUpdaterHandler); router.get('/_/grp/:grp', balanceUpdaterHandler); router.get('/', totalSupplyHandler); router.get('/balances/liquid/:address', liquidBalanceHandler); From 1aa7d8bc56b3b5f66345fda8f1068174edcca900 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 30 Sep 2022 06:45:16 -0300 Subject: [PATCH 035/224] Made group be random on cron --- src/index.ts | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/index.ts b/src/index.ts index abe09e3a..b3a784db 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,10 +14,7 @@ import { updateGroupBalances } from "./handlers/cron"; addEventListener('scheduled', (event: any) => { console.log(`triggering scheduled account balance update`) - event.waitUntil(updateGroupBalances(1, event)); - event.waitUntil(updateGroupBalances(2, event)); - event.waitUntil(updateGroupBalances(3, event)); - event.waitUntil(updateGroupBalances(4, event)); + event.waitUntil(updateGroupBalances(getRandomGroup(), event)); }) addEventListener('fetch', (event: FetchEvent) => { @@ -47,3 +44,25 @@ function registerRoutes(router: Router) { function handleError(error: Error): Response { return new Response(error.message || 'Server Error', { status: 500 }) } + +function getRandomGroup(): number { + const hour = new Date().getHours() + + if (hour > 0 && hour < 6) { + return 1; + } + + if (hour >= 6 && hour < 12) { + return 2; + } + + if (hour >= 12 && hour < 18) { + return 2; + } + + if (hour >= 18 && hour < 24) { + return 2; + } + + throw new Error("invalid hour for group"); +} From 23740925667c69c3a1441773cbff9e94713cef25 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 30 Sep 2022 06:54:34 -0300 Subject: [PATCH 036/224] Simplified record on KV --- src/helpers/balance.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 3050d458..5a1e540c 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,6 +1,7 @@ import { NodeApi } from "../api/nodeApi"; import { GraphQLClient } from "./graphql"; import { BigDipperApi } from "../api/bigDipperApi"; +import { total_balance_ncheq } from "./node"; export async function updateBalance(node_api: NodeApi, addr: string, grpN: number): Promise { const gql_client = new GraphQLClient(GRAPHQL_API); @@ -17,7 +18,10 @@ export async function updateBalance(node_api: NodeApi, addr: string, grpN: numbe if (cachedAccount !== undefined) { console.log(`account "${addr}" found in cache: ${cachedAccount}`) - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, JSON.stringify(account)) + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, JSON.stringify({ + totalBalance: total_balance_ncheq(account), + })) + console.log(`account "${addr}" balance updated. (res=${JSON.stringify(account)})`) return new Response(JSON.stringify(account)); From a2d2395c046f461e58a6778e89b7194e13427c9f Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 30 Sep 2022 06:58:33 -0300 Subject: [PATCH 037/224] Better naming and place for cron helper --- src/handlers/balanceUpdater.ts | 2 +- .../cron.ts => helpers/balanceGroup.ts} | 20 +++++++++---------- src/index.ts | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) rename src/{handlers/cron.ts => helpers/balanceGroup.ts} (63%) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index d71c50fc..2b438166 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -1,5 +1,5 @@ import { Request } from "itty-router"; -import { updateGroupBalances } from "./cron"; +import { updateGroupBalances } from "../helpers/balanceGroup"; export async function handler(request: Request): Promise { const grp = request.params?.['grp']; diff --git a/src/handlers/cron.ts b/src/helpers/balanceGroup.ts similarity index 63% rename from src/handlers/cron.ts rename to src/helpers/balanceGroup.ts index fecf5f07..f207fea0 100644 --- a/src/handlers/cron.ts +++ b/src/helpers/balanceGroup.ts @@ -1,5 +1,5 @@ import { NodeApi } from "../api/nodeApi"; -import { updateBalance } from "../helpers/balance"; +import { updateBalance } from "./balance"; import { Account } from "../types/bigDipper"; export async function updateGroupBalances(group: number, event: Event) { @@ -7,30 +7,28 @@ export async function updateGroupBalances(group: number, event: Event) { let balances: { account: String, balances: Account } [] = []; try { - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${group}:` }); + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); console.log(`found ${cached.keys.length} cached accounts`) for (const key of cached.keys) { const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name) if (found) { - let addr: string; - if (key.name.startsWith("grp_")) { + const parts = key.name.split(':') + let addr = parts[1] + + if (addr.startsWith("delayed:")) { const parts = key.name.split(':') addr = parts[1] - } else { - addr = key.name } - + const res = await updateBalance(node_api, addr, group) if (res !== undefined) { const data = await res.json() as Account; - if (!data.error) { - console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(data)})`) - balances.push({ account: addr, balances: data }) - } + console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(data)})`) + balances.push({ account: addr, balances: data }) } } } diff --git a/src/index.ts b/src/index.ts index b3a784db..3ed596b6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ import { handler as delegatorCountHandler } from './handlers/delegatorCount'; import { handler as totalDelegatorsHandler } from './handlers/totalDelegators'; import { handler as totalStakedCoinsHandler } from "./handlers/totalStakedCoins"; import { handler as balanceUpdaterHandler } from "./handlers/balanceUpdater"; -import { updateGroupBalances } from "./handlers/cron"; +import { updateGroupBalances } from "./helpers/balanceGroup"; addEventListener('scheduled', (event: any) => { console.log(`triggering scheduled account balance update`) From 9403d2f85df18dd7a3419a0282d3e4347de9f488 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 30 Sep 2022 07:04:48 -0300 Subject: [PATCH 038/224] Better error handling --- src/handlers/balanceUpdater.ts | 11 ++++------- src/helpers/balance.ts | 14 +++++++------- src/helpers/balanceGroup.ts | 14 ++++++-------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index 2b438166..a57a7044 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -4,13 +4,10 @@ import { updateGroupBalances } from "../helpers/balanceGroup"; export async function handler(request: Request): Promise { const grp = request.params?.['grp']; - console.log(`updating all account balances (group: ${grp})`) - const res = await updateGroupBalances(Number(grp), {} as Event) - if (res !== undefined) { - return res + if (grp !== null) { + console.log(`updating all account balances (group: ${grp})`) + return updateGroupBalances(Number(grp), {} as Event) } - return new Response(JSON.stringify({ - error: new Error("this endpoint should be called for all accounts") - })) + return new Response("group is missing") } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 5a1e540c..97e4e15e 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -2,14 +2,15 @@ import { NodeApi } from "../api/nodeApi"; import { GraphQLClient } from "./graphql"; import { BigDipperApi } from "../api/bigDipperApi"; import { total_balance_ncheq } from "./node"; +import { Account } from "../types/bigDipper"; -export async function updateBalance(node_api: NodeApi, addr: string, grpN: number): Promise { +export async function updateCachedBalance(node_api: NodeApi, addr: string, grpN: number): Promise { const gql_client = new GraphQLClient(GRAPHQL_API); const bd_api = new BigDipperApi(gql_client); const account = await bd_api.get_account(addr); if (!account) { - return new Response(JSON.stringify({ error: "Account not found" })) + throw new Error(`Account not found for address "${addr}"`) } try { @@ -24,13 +25,12 @@ export async function updateBalance(node_api: NodeApi, addr: string, grpN: numbe console.log(`account "${addr}" balance updated. (res=${JSON.stringify(account)})`) - return new Response(JSON.stringify(account)); + return account; } - return new Response(JSON.stringify({ error: "Account not found" })); + return null; } catch (e: any) { - console.error(e) - console.log(new Map(e)) - return new Response(JSON.stringify({ error: e })) + console.error(new Map(e)) + throw e; } } diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index f207fea0..5445357d 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -1,10 +1,10 @@ import { NodeApi } from "../api/nodeApi"; -import { updateBalance } from "./balance"; +import { updateCachedBalance } from "./balance"; import { Account } from "../types/bigDipper"; export async function updateGroupBalances(group: number, event: Event) { let node_api = new NodeApi(REST_API); - let balances: { account: String, balances: Account } [] = []; + let balances: { account: Account } [] = []; try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); @@ -21,14 +21,12 @@ export async function updateGroupBalances(group: number, event: Event) { const parts = key.name.split(':') addr = parts[1] } - - const res = await updateBalance(node_api, addr, group) - if (res !== undefined) { - const data = await res.json() as Account; + const account = await updateCachedBalance(node_api, addr, group) - console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(data)})`) - balances.push({ account: addr, balances: data }) + if (account) { + console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(account)})`) + balances.push({ account: account }) } } } From a48af48b1b6499ff4561b2ba0fd31ce3f54cc2b1 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Fri, 30 Sep 2022 07:12:35 -0300 Subject: [PATCH 039/224] Removed unused code --- src/handlers/balanceUpdater.ts | 7 ++++++- src/handlers/circulatingSupply.ts | 27 ++++++++++----------------- src/helpers/balance.ts | 9 +++++---- src/helpers/balanceGroup.ts | 5 ++--- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceUpdater.ts index a57a7044..948e91fb 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceUpdater.ts @@ -6,7 +6,12 @@ export async function handler(request: Request): Promise { if (grp !== null) { console.log(`updating all account balances (group: ${grp})`) - return updateGroupBalances(Number(grp), {} as Event) + try { + return updateGroupBalances(Number(grp), {} as Event) + } catch (e: any) { + console.log(new Map(e)) + return new Response(JSON.stringify(new Map(e))) + } } return new Response("group is missing") diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 3ca480e3..ab2b3818 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -2,9 +2,8 @@ import { GraphQLClient } from "../helpers/graphql"; import { BigDipperApi } from "../api/bigDipperApi"; import { Request } from "itty-router"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; -import { total_balance_ncheq } from "../helpers/node"; -async function get_circulating_supply(circulating_supply_watchlist: string[]): Promise { +async function get_circulating_supply(): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); @@ -12,23 +11,20 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); try { - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: 'grp_' - }) - + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list() console.log(`found ${cached.keys.length} cached items`) let non_circulating_supply_ncheq = 0; - for (const account of cached.keys) { - console.log(`looking for account: "${JSON.stringify(account.name)}" in cache`) - let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(account.name); + for (const r of cached.keys) { + console.log(`looking for account: ${JSON.stringify(r.name)} in cache`) + let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(r.name); - if (cachedFound !== undefined) { - const data = JSON.parse(cachedFound) + if (cachedFound !== null) { + const data: { totalBalance: number | null } = JSON.parse(cachedFound) - if (typeof data === "object") { + if (data.totalBalance !== null) { console.log(`found cache entry: ${cachedFound}`) - non_circulating_supply_ncheq += total_balance_ncheq(data); + non_circulating_supply_ncheq += data.totalBalance; } } } @@ -47,10 +43,7 @@ async function get_circulating_supply(circulating_supply_watchlist: string[]): P } export async function handler(request: Request): Promise { - let addresses_to_exclude: string[] = (await CIRCULATING_SUPPLY_WATCHLIST.list()).keys.filter(k => !k.name.startsWith("grp_")); - - let circulating_supply = await get_circulating_supply(addresses_to_exclude); - console.log(`circulating_supply=${circulating_supply}`) + let circulating_supply = await get_circulating_supply(); return new Response(ncheq_to_cheq_fixed(circulating_supply)); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 97e4e15e..91066be0 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -19,11 +19,12 @@ export async function updateCachedBalance(node_api: NodeApi, addr: string, grpN: if (cachedAccount !== undefined) { console.log(`account "${addr}" found in cache: ${cachedAccount}`) - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, JSON.stringify({ - totalBalance: total_balance_ncheq(account), - })) + const totalBalance = total_balance_ncheq(account); + const data = JSON.stringify({ totalBalance }); - console.log(`account "${addr}" balance updated. (res=${JSON.stringify(account)})`) + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data) + + console.log(`account "${addr}" balance updated. (${data})`) return account; } diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index 5445357d..04426a41 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -17,9 +17,8 @@ export async function updateGroupBalances(group: number, event: Event) { const parts = key.name.split(':') let addr = parts[1] - if (addr.startsWith("delayed:")) { - const parts = key.name.split(':') - addr = parts[1] + if (key.name.startsWith("delayed:")) { + addr = parts[2] } const account = await updateCachedBalance(node_api, addr, group) From 4b79e2ff50ca926d020019e834e5d375011d3f40 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Fri, 30 Sep 2022 08:05:08 -0300 Subject: [PATCH 040/224] Better yet error handling --- src/handlers/circulatingSupply.ts | 2 +- src/helpers/balance.ts | 2 +- src/helpers/balanceGroup.ts | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index ab2b3818..a2251d9d 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -19,7 +19,7 @@ async function get_circulating_supply(): Promise { console.log(`looking for account: ${JSON.stringify(r.name)} in cache`) let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(r.name); - if (cachedFound !== null) { + if (cachedFound) { const data: { totalBalance: number | null } = JSON.parse(cachedFound) if (data.totalBalance !== null) { diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 91066be0..29eb01ad 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -29,7 +29,7 @@ export async function updateCachedBalance(node_api: NodeApi, addr: string, grpN: return account; } - return null; + return account; } catch (e: any) { console.error(new Map(e)) throw e; diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index 04426a41..8478c3be 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -23,7 +23,7 @@ export async function updateGroupBalances(group: number, event: Event) { const account = await updateCachedBalance(node_api, addr, group) - if (account) { + if (account !== null) { console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(account)})`) balances.push({ account: account }) } @@ -31,8 +31,8 @@ export async function updateGroupBalances(group: number, event: Event) { } return new Response(JSON.stringify(balances)); - } catch (e) { - console.error(e) + } catch (e: any) { + console.error(new Map(e)) throw e; } } From 30d1978595594c5611a1a63be5d7e514016b4ba8 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Fri, 30 Sep 2022 08:15:41 -0300 Subject: [PATCH 041/224] Fixed error issue returning bad response --- src/api/bigDipperApi.ts | 2 +- .../{balanceUpdater.ts => balanceGroup.ts} | 10 ++--- src/handlers/circulatingSupply.ts | 20 ++++++--- src/helpers/balance.ts | 22 +++++----- src/helpers/balanceGroup.ts | 44 +++++++++---------- src/helpers/graphql.ts | 2 +- src/index.ts | 2 +- 7 files changed, 53 insertions(+), 49 deletions(-) rename src/handlers/{balanceUpdater.ts => balanceGroup.ts} (63%) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 4c00c400..feb1fb05 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -48,7 +48,7 @@ export class BigDipperApi { return resp.data as Account } catch (e: any) { - console.error(new Map(e)) + console.error(e) return null; } } diff --git a/src/handlers/balanceUpdater.ts b/src/handlers/balanceGroup.ts similarity index 63% rename from src/handlers/balanceUpdater.ts rename to src/handlers/balanceGroup.ts index 948e91fb..65109855 100644 --- a/src/handlers/balanceUpdater.ts +++ b/src/handlers/balanceGroup.ts @@ -6,12 +6,10 @@ export async function handler(request: Request): Promise { if (grp !== null) { console.log(`updating all account balances (group: ${grp})`) - try { - return updateGroupBalances(Number(grp), {} as Event) - } catch (e: any) { - console.log(new Map(e)) - return new Response(JSON.stringify(new Map(e))) - } + const result = await updateGroupBalances(Number(grp), {} as Event) + + console.log({ result }) + return new Response(JSON.stringify(result)); } return new Response("group is missing") diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index a2251d9d..a7800eda 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -2,6 +2,8 @@ import { GraphQLClient } from "../helpers/graphql"; import { BigDipperApi } from "../api/bigDipperApi"; import { Request } from "itty-router"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; +import { total_balance_ncheq } from "../helpers/node"; +import { Account } from "../types/bigDipper"; async function get_circulating_supply(): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); @@ -14,16 +16,22 @@ async function get_circulating_supply(): Promise { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list() console.log(`found ${cached.keys.length} cached items`) - let non_circulating_supply_ncheq = 0; + let non_circulating_supply_ncheq = Number(0); for (const r of cached.keys) { - console.log(`looking for account: ${JSON.stringify(r.name)} in cache`) - let cachedFound = await CIRCULATING_SUPPLY_WATCHLIST.get(r.name); + console.log(`looking for account: ${r.name} in cache`) + let data: any = await CIRCULATING_SUPPLY_WATCHLIST.get(r.name, { type: "json" }); + + if (data !== null) { + if (data.totalBalance === undefined) { + const balance = total_balance_ncheq(JSON.parse(data) as Account) + data = JSON.stringify({ totalBalance: balance }) + console.log(`updating bad cache entry: ${JSON.stringify(data)} totalBalance=${data.totalBalance} data=${JSON.stringify(data)}`) + await CIRCULATING_SUPPLY_WATCHLIST.put(r.name, data) + } - if (cachedFound) { - const data: { totalBalance: number | null } = JSON.parse(cachedFound) + console.log(`found cache entry: ${JSON.stringify(data)} totalBalance=${data.totalBalance}`) if (data.totalBalance !== null) { - console.log(`found cache entry: ${cachedFound}`) non_circulating_supply_ncheq += data.totalBalance; } } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 29eb01ad..a6aa66fa 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -14,24 +14,24 @@ export async function updateCachedBalance(node_api: NodeApi, addr: string, grpN: } try { - const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`) + const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`, { type: "json" }) - if (cachedAccount !== undefined) { - console.log(`account "${addr}" found in cache: ${cachedAccount}`) + // if (cachedAccount !== undefined) { + console.log(`account "${addr}" found in cache: ${JSON.stringify(cachedAccount)}`) - const totalBalance = total_balance_ncheq(account); - const data = JSON.stringify({ totalBalance }); + const totalBalance = total_balance_ncheq(account); + const data = JSON.stringify({ totalBalance: totalBalance }); - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data) + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data) - console.log(`account "${addr}" balance updated. (${data})`) + console.log(`account "${addr}" balance updated. (${data})`) - return account; - } + return account; + // } return account; } catch (e: any) { - console.error(new Map(e)) - throw e; + console.error(e) + return null; } } diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index 8478c3be..ee9f6e58 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -1,38 +1,36 @@ -import { NodeApi } from "../api/nodeApi"; -import { updateCachedBalance } from "./balance"; import { Account } from "../types/bigDipper"; +import { updateCachedBalance } from "./balance"; +import { NodeApi } from "../api/nodeApi"; export async function updateGroupBalances(group: number, event: Event) { let node_api = new NodeApi(REST_API); let balances: { account: Account } [] = []; - try { - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${group}:` }); + console.log(`found ${cached.keys.length} cached accounts`) - console.log(`found ${cached.keys.length} cached accounts`) - for (const key of cached.keys) { - const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name) + for (const key of cached.keys) { + console.log(`searching ${key.name}`) + const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name) - if (found) { - const parts = key.name.split(':') - let addr = parts[1] + if (found) { + const parts = key.name.split(':') + let addr = parts[1] + let grpN = parts[0].split("_")[1] - if (key.name.startsWith("delayed:")) { - addr = parts[2] - } + if (key.name.includes("delayed:")) { + addr = parts[2] + } + + console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`) - const account = await updateCachedBalance(node_api, addr, group) + const account = await updateCachedBalance(node_api, addr, grpN) + console.log(`updated account ${JSON.stringify(account)}`) - if (account !== null) { - console.log(`updating account (grp_${group}:${addr}) balance (${JSON.stringify(account)})`) - balances.push({ account: account }) - } + if (account !== null) { + console.log(`updating account (grp_${grpN}:${addr}) balance (${JSON.stringify(account)})`) + balances.push({ account: account }) } } - - return new Response(JSON.stringify(balances)); - } catch (e: any) { - console.error(new Map(e)) - throw e; } } diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index a7e6724b..e16a5d4c 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -16,7 +16,7 @@ export class GraphQLClient { let json: { errors: any } = await resp.json() if (json.errors) { - throw new Error(`query failed: ${json.errors}`) + throw new Error(`query failed: ${JSON.stringify(json.errors)}`) } return json as T; diff --git a/src/index.ts b/src/index.ts index 3ed596b6..a11cb0b0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import { handler as vestedBalanceHandler } from "./handlers/vestedBalance"; import { handler as delegatorCountHandler } from './handlers/delegatorCount'; import { handler as totalDelegatorsHandler } from './handlers/totalDelegators'; import { handler as totalStakedCoinsHandler } from "./handlers/totalStakedCoins"; -import { handler as balanceUpdaterHandler } from "./handlers/balanceUpdater"; +import { handler as balanceUpdaterHandler } from "./handlers/balanceGroup"; import { updateGroupBalances } from "./helpers/balanceGroup"; addEventListener('scheduled', (event: any) => { From a1b5229e12544caac16d984d0f33261491a6d4a0 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Sun, 2 Oct 2022 08:55:49 -0300 Subject: [PATCH 042/224] Clean-up of addresses --- src/helpers/balanceGroup.ts | 13 ++++++++++--- src/index.ts | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index ee9f6e58..5d519cb4 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -1,14 +1,20 @@ -import { Account } from "../types/bigDipper"; import { updateCachedBalance } from "./balance"; import { NodeApi } from "../api/nodeApi"; +import { Account } from "../types/bigDipper"; export async function updateGroupBalances(group: number, event: Event) { let node_api = new NodeApi(REST_API); let balances: { account: Account } [] = []; const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${group}:` }); + // const cached = await import ("../../addresses.json"); console.log(`found ${cached.keys.length} cached accounts`) + + for (const key of cached.keys) { + CIRCULATING_SUPPLY_WATCHLIST.delete(key.name) + } + for (const key of cached.keys) { console.log(`searching ${key.name}`) const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name) @@ -16,7 +22,7 @@ export async function updateGroupBalances(group: number, event: Event) { if (found) { const parts = key.name.split(':') let addr = parts[1] - let grpN = parts[0].split("_")[1] + let grpN = Number(parts[0].split("_")[1]) if (key.name.includes("delayed:")) { addr = parts[2] @@ -25,7 +31,6 @@ export async function updateGroupBalances(group: number, event: Event) { console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`) const account = await updateCachedBalance(node_api, addr, grpN) - console.log(`updated account ${JSON.stringify(account)}`) if (account !== null) { console.log(`updating account (grp_${grpN}:${addr}) balance (${JSON.stringify(account)})`) @@ -33,4 +38,6 @@ export async function updateGroupBalances(group: number, event: Event) { } } } + + return balances } diff --git a/src/index.ts b/src/index.ts index a11cb0b0..713688fa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,7 +24,6 @@ addEventListener('fetch', (event: FetchEvent) => { }) function registerRoutes(router: Router) { - router.get('/_/grp/:grp', balanceUpdaterHandler); router.get('/', totalSupplyHandler); router.get('/balances/liquid/:address', liquidBalanceHandler); router.get('/balances/total/:address', totalBalanceHandler); @@ -35,6 +34,7 @@ function registerRoutes(router: Router) { router.get('/supply/circulating', circulatingSupplyHandler); router.get('/supply/staked', totalStakedCoinsHandler); router.get('/supply/total', totalSupplyHandler); + router.get('/_/grp/:grp', balanceUpdaterHandler); // 404 for all other requests From 0e821e71f8f6470be4aa6f40857eef7d3f3a4946 Mon Sep 17 00:00:00 2001 From: drgomesp Date: Sun, 2 Oct 2022 09:04:00 -0300 Subject: [PATCH 043/224] Reprocessing addresses --- addresses.json | 286 ++++++++++++++++++++++++++++++++++++ src/api/bigDipperApi.ts | 2 +- src/helpers/balance.ts | 2 +- src/helpers/balanceGroup.ts | 21 +-- 4 files changed, 295 insertions(+), 16 deletions(-) create mode 100644 addresses.json diff --git a/addresses.json b/addresses.json new file mode 100644 index 00000000..2f0d95aa --- /dev/null +++ b/addresses.json @@ -0,0 +1,286 @@ +{ + "keys": [ + { + "name": "grp_3:cheqd1t0qhk7kqe6vv36alehrwn2ht5e0dc8wh2uzkr6" + }, + { + "name": "grp_3:cheqd1t0vw4kha8ffw0sfsd2k56287q73zwnca5gr7l2" + }, + { + "name": "grp_3:cheqd1t3gsdl2uux0rmt3acn8qursc9sarhtm6mpa3hq" + }, + { + "name": "grp_3:cheqd1t5pl7x25egk2xcfu2kyg4a7ga4hyg2g5anv5hh" + }, + { + "name": "grp_3:cheqd1tgtgmksy2de42nzahk9kwml9crf36r735xwazj" + }, + { + "name": "grp_3:cheqd1thhqk2q4s8h3ugwjm4h2v09gxaturfzkyjeux3" + }, + { + "name": "grp_3:cheqd1tml0chnp3mee88cyk63la3ctj9ndu4vtl2e2m7" + }, + { + "name": "grp_3:cheqd1tpvhkuvn8je2qdghexfsgeg0q37zcqej06m30x" + }, + { + "name": "grp_3:cheqd1tqpxjdnxcjqswrcq5slz64wrh7t5j6jle7gwfh" + }, + { + "name": "grp_3:cheqd1tt5p2xwz3xv2d4wjkegycu6pauv3jxyta3vj0r" + }, + { + "name": "grp_3:cheqd1tx65lajxvzwmvfxwvev23kxac2v3zj3jg0hrcw" + }, + { + "name": "grp_3:cheqd1u3uwj2rvxt6ytweypemcc35vy80zwlnx7ly6wt" + }, + { + "name": "grp_3:cheqd1ucxsrv5yyvxhjk2cu3gl54l94dru7apgwrlk9q" + }, + { + "name": "grp_3:cheqd1uvjqw3wadfjycznpe0z8u36aqnecndtzlwgf7s" + }, + { + "name": "grp_3:cheqd1va3slq5k97a008gxncn9eavvf4mt84xcmej65l" + }, + { + "name": "grp_3:cheqd1vex9phvwcwr785k2jal7tv2pklysqx94kzay8a" + }, + { + "name": "grp_3:cheqd1vh0rar0jgyx8xeett26nv77jnvxr99k8q8qa6q" + }, + { + "name": "grp_3:cheqd1vj9qtf9fy00q8xnwxfwsswtr24ajl8a592hyp8" + }, + { + "name": "grp_3:cheqd1vl23k3x73wzq4t7x3sh54n5f40kpse32pleka6" + }, + { + "name": "grp_3:cheqd1vlu2azc2fs9p5zffyeefth0l79eygwj8vmt2r6" + }, + { + "name": "grp_3:cheqd1vusv4s233mw867ua63m7upjxzgcyqhwhumqe6d" + }, + { + "name": "grp_3:cheqd1vxfyppcnu6lz72mdc706mlv7kx8434jaknhgt6" + }, + { + "name": "grp_3:cheqd1w5xsmev2zv59yhtk5nsl6xrdwpu588k5su6xs0" + }, + { + "name": "grp_3:cheqd1wsf8ns8gmcq76nes484pt46c9ru9c9mdmwm7sc" + }, + { + "name": "grp_3:cheqd1wwwncrnde9myewpn8j00xeyt4hfs3sqmkyasqm" + }, + { + "name": "grp_3:cheqd1wznaxsqya0meesp9f8xrknnreuszfd8kup3mq5" + }, + { + "name": "grp_3:cheqd1wzyfq9ujg7vp3whn7nh7w5ner8p7ghk2pxw0g5" + }, + { + "name": "grp_3:cheqd1x7ku9c2x2j263yaj48ct9nwjark75f0hhgmpvg" + }, + { + "name": "grp_3:cheqd1x9dc5enk7f77apgscc3emep5nz0fapxm4w9r9m" + }, + { + "name": "grp_3:cheqd1xqgm3nw0uh4k4zj0u4l2ljfqslrfumff5syu49" + }, + { + "name": "grp_3:cheqd1xqxz5wneg2hqg9tugzuv8v8wgyr2pl32v2h3hl" + }, + { + "name": "grp_3:cheqd1xs7kvk7n7m6ca6yw7ysxxfulwauc2s46z2w283" + }, + { + "name": "grp_3:cheqd1xusdhhtlqk0vetgqg7nu39j9q62zq63q22k463" + }, + { + "name": "grp_3:cheqd1xwnu2mm96uexuergqk0kqt4ds84x9zsjyne49e" + }, + { + "name": "grp_3:cheqd1xyyygaxx384k07sv3rk9zj7dezuzf250pahr0w" + }, + { + "name": "grp_3:cheqd1y6ekzultqshf8xwtv46mlzfquj6ys3vmncascq" + }, + { + "name": "grp_3:cheqd1y8edvwl536rxuqdznh48mh6mg648gf9a3xx2s9" + }, + { + "name": "grp_3:cheqd1ygwl8m54cpenq7t5v5p0y9xau3gmg873fqlqag" + }, + { + "name": "grp_3:cheqd1ylfukxalf0zr2wdma0gx33e9fccha94qr59rj4" + }, + { + "name": "grp_3:cheqd1yp52vx7sjv2dhhfqe2n52ntr0av39lye9fx5gl" + }, + { + "name": "grp_3:cheqd1yqj0s0htaseqc9prkcsmucme3ef5fsyq5mkqrd" + }, + { + "name": "grp_3:cheqd1zh8g8lw0zhp0qk0fxjq9n5p2yjc4xgpca5gzds" + }, + { + "name": "grp_3:cheqd1zhqu2pzpsv0x5afmw3tm5579pqxy59msh8ytj3" + }, + { + "name": "grp_3:cheqd1zhw4qtrd04kp7zz329zjlwy0s3vjunxua6rrqd" + }, + { + "name": "grp_3:cheqd1zm2pz9gj4vsl278vydapx0x8e3fguvxalg7n98" + }, + { + "name": "grp_3:cheqd1zmxl82zfgqsc0efuyhateaen8mzp7ngkaw4qxz" + }, + { + "name": "grp_3:cheqd1zu4mm0lzhm7az4fwc6d28pkt2vvpte5ye45rgv" + }, + { + "name": "grp_3:cheqd1zym5ync279m2sxx99dhtv4s00apr08lzw2g8g8" + }, + { + "name": "grp_3:cheqd1zz0gck564g3dvaxctccc5kgxw2sgtes7p2led6" + }, + { + "name": "grp_4:delayed:cheqd10vmxqqnc9j7axh808ymyz8t8kneux5n8xmmhq9" + }, + { + "name": "grp_4:delayed:cheqd14c5zvsf74uayyp90mav49gx2aj250s8u8yv7y9" + }, + { + "name": "grp_4:delayed:cheqd15yd3fnn3ev5xeacazejgejd5jxrnpusg8mgu8h" + }, + { + "name": "grp_4:delayed:cheqd15zw9j5e3l4l3k6fex8e7hznzmf0mpz5vcwlrt3" + }, + { + "name": "grp_4:delayed:cheqd17n6mefp8vhyrtvavc9fd9hg0hx624skgdjtrpz" + }, + { + "name": "grp_4:delayed:cheqd184z8z58dmax77csg0a9tneku0sqskeqymx5hxw" + }, + { + "name": "grp_4:delayed:cheqd18jc49zfkew8aj8zmxjyrk3nr8asqu8jxg56fwz" + }, + { + "name": "grp_4:delayed:cheqd198yncktukks0aqtyk4vq0hw20v67ffr0rm8cla" + }, + { + "name": "grp_4:delayed:cheqd1a0vumnau34trnvjpnqr4ywwaqd89l4s7ytwkk5" + }, + { + "name": "grp_4:delayed:cheqd1ajefdpfng3l66gr2j3ygvh6efrr6twrd3968rk" + }, + { + "name": "grp_4:delayed:cheqd1c5wva37ufnpdypxkxeya9kawrzgay6evnra50u" + }, + { + "name": "grp_4:delayed:cheqd1cpj9vd5e3dyvyj0nds3hanz30zafakdc4h00wj" + }, + { + "name": "grp_4:delayed:cheqd1crws5sz8tdl37v40tjn0lmlu6ztjhxar9jklqz" + }, + { + "name": "grp_4:delayed:cheqd1dy69x2g8vj0sdnutr29lt2wqgxqw98pcxltuu5" + }, + { + "name": "grp_4:delayed:cheqd1eraud0nswfglksl9tl8e5dlemrq5eualjftv3z" + }, + { + "name": "grp_4:delayed:cheqd1ewhshdq9cdhxwx8vrnnffhfe8qw3szryqv4gu7" + }, + { + "name": "grp_4:delayed:cheqd1exe54lymn5mhpjs0vj6yppnecdmtfss2qcd4cq" + }, + { + "name": "grp_4:delayed:cheqd1g3379fkvfc6jgxa66dswzsmzeguryv6zkxca87" + }, + { + "name": "grp_4:delayed:cheqd1gz3suxvpqrdwntnn7r95eerwa3ura92kp4tm8c" + }, + { + "name": "grp_4:delayed:cheqd1jkzd7sn6dzn54tl0y4ztw7n68r0p4aw53f5mqx" + }, + { + "name": "grp_4:delayed:cheqd1k3vc8xplwtqpyt55ysx085lhr09xxvhuq9ddra" + }, + { + "name": "grp_4:delayed:cheqd1kaascflyltj3pp54hwarnat6kmwtw2zvllsqdc" + }, + { + "name": "grp_4:delayed:cheqd1kq5atufq4ryhdld8s7cmy2735uxhh6wdakj9cj" + }, + { + "name": "grp_4:delayed:cheqd1kysvc7hq8944jq68yjg7hqxud66r3ajx8uv5s8" + }, + { + "name": "grp_4:delayed:cheqd1la5de5x52qw8vspwj70g6c6mem02al8sf4z4sm" + }, + { + "name": "grp_4:delayed:cheqd1lp0lq9mhc22y42znga5mvjngjpdj4pgvgzs950" + }, + { + "name": "grp_4:delayed:cheqd1lvgq858vk9hs9stfkkv0dhhvnwv5eczjx4rkjw" + }, + { + "name": "grp_4:delayed:cheqd1m5r4emnpmgnl0xgptk4f4pkrqmnlrzpnhs8aqu" + }, + { + "name": "grp_4:delayed:cheqd1mn83wm6f43f9ls6lsju8w2x4z4rg4rcvmcsq7z" + }, + { + "name": "grp_4:delayed:cheqd1nn7rcyvfd3zqhumuwwj05wvdh4x28f2qs3sex5" + }, + { + "name": "grp_4:delayed:cheqd1ppuya3ezvs2jq7n2e0ygec6r3uhx0gea6hl9vp" + }, + { + "name": "grp_4:delayed:cheqd1pw0z58t0jrtexp26694ycrdgv6ycmt5fcy00vg" + }, + { + "name": "grp_4:delayed:cheqd1q4lkgp3psk2fdxt2pgr0lujyj5fk4lz2wlz8dy" + }, + { + "name": "grp_4:delayed:cheqd1q6wwcqf902ygu8z7dcyhh8ca0u6a5g4kns6he8" + }, + { + "name": "grp_4:delayed:cheqd1qxr4jdgug8dve9kq6vc6pwdhpuyvlsql85xx0z" + }, + { + "name": "grp_4:delayed:cheqd1r2y0ffh5xgkrnzelygz35mprjejaatnah9untf" + }, + { + "name": "grp_4:delayed:cheqd1s0sngupwnkfv84r079x5ct74j8nnxgjfp89p54" + }, + { + "name": "grp_4:delayed:cheqd1ssge5ykw4hvu4jy6eeen7hyxnw72mds6mzj79h" + }, + { + "name": "grp_4:delayed:cheqd1ssjkeq5k385fydn9n7qvcp74pqkxgl8r8eczfq" + }, + { + "name": "grp_4:delayed:cheqd1sueav9pmjy5k8crpg2nuakyssd86kcrrlc0qre" + }, + { + "name": "grp_4:delayed:cheqd1t62uzdkyaw27v59ufyftzccl707k8a6rt9kcgn" + }, + { + "name": "grp_4:delayed:cheqd1w53yz6ylpraykng3sw6r8lunuar7lteavel3x0" + }, + { + "name": "grp_4:delayed:cheqd1x6srspstwx3pdta64d99en9radmaztms6lj0te" + }, + { + "name": "grp_4:delayed:cheqd1y3fvu2ychq7247fkwfna0nn6exl9w6u009tnh7" + }, + { + "name": "grp_4:delayed:cheqd1z8hvr2drwgnfqudkrqhwvdv5p40h8maw7jg92g" + } + ] +} diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index feb1fb05..4c00c400 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -48,7 +48,7 @@ export class BigDipperApi { return resp.data as Account } catch (e: any) { - console.error(e) + console.error(new Map(e)) return null; } } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index a6aa66fa..454459e9 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -31,7 +31,7 @@ export async function updateCachedBalance(node_api: NodeApi, addr: string, grpN: return account; } catch (e: any) { - console.error(e) + console.error(new Map(e)) return null; } } diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index 5d519cb4..a14e4903 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -10,24 +10,17 @@ export async function updateGroupBalances(group: number, event: Event) { // const cached = await import ("../../addresses.json"); console.log(`found ${cached.keys.length} cached accounts`) - for (const key of cached.keys) { - CIRCULATING_SUPPLY_WATCHLIST.delete(key.name) - } + const parts = key.name.split(':') + let addr = parts[1] + let grpN = Number(parts[0].split("_")[1]) - for (const key of cached.keys) { - console.log(`searching ${key.name}`) - const found = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name) + if (key.name.includes("delayed:")) { + addr = parts[2] + } + const found = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`) if (found) { - const parts = key.name.split(':') - let addr = parts[1] - let grpN = Number(parts[0].split("_")[1]) - - if (key.name.includes("delayed:")) { - addr = parts[2] - } - console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`) const account = await updateCachedBalance(node_api, addr, grpN) From 84fde0bacc379f110803967fe405c4abb27973b0 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Sun, 2 Oct 2022 18:38:21 -0300 Subject: [PATCH 044/224] Removed static account addresses --- addresses.json | 286 ------------------------------------ src/api/bigDipperApi.ts | 2 +- src/helpers/balanceGroup.ts | 2 +- 3 files changed, 2 insertions(+), 288 deletions(-) delete mode 100644 addresses.json diff --git a/addresses.json b/addresses.json deleted file mode 100644 index 2f0d95aa..00000000 --- a/addresses.json +++ /dev/null @@ -1,286 +0,0 @@ -{ - "keys": [ - { - "name": "grp_3:cheqd1t0qhk7kqe6vv36alehrwn2ht5e0dc8wh2uzkr6" - }, - { - "name": "grp_3:cheqd1t0vw4kha8ffw0sfsd2k56287q73zwnca5gr7l2" - }, - { - "name": "grp_3:cheqd1t3gsdl2uux0rmt3acn8qursc9sarhtm6mpa3hq" - }, - { - "name": "grp_3:cheqd1t5pl7x25egk2xcfu2kyg4a7ga4hyg2g5anv5hh" - }, - { - "name": "grp_3:cheqd1tgtgmksy2de42nzahk9kwml9crf36r735xwazj" - }, - { - "name": "grp_3:cheqd1thhqk2q4s8h3ugwjm4h2v09gxaturfzkyjeux3" - }, - { - "name": "grp_3:cheqd1tml0chnp3mee88cyk63la3ctj9ndu4vtl2e2m7" - }, - { - "name": "grp_3:cheqd1tpvhkuvn8je2qdghexfsgeg0q37zcqej06m30x" - }, - { - "name": "grp_3:cheqd1tqpxjdnxcjqswrcq5slz64wrh7t5j6jle7gwfh" - }, - { - "name": "grp_3:cheqd1tt5p2xwz3xv2d4wjkegycu6pauv3jxyta3vj0r" - }, - { - "name": "grp_3:cheqd1tx65lajxvzwmvfxwvev23kxac2v3zj3jg0hrcw" - }, - { - "name": "grp_3:cheqd1u3uwj2rvxt6ytweypemcc35vy80zwlnx7ly6wt" - }, - { - "name": "grp_3:cheqd1ucxsrv5yyvxhjk2cu3gl54l94dru7apgwrlk9q" - }, - { - "name": "grp_3:cheqd1uvjqw3wadfjycznpe0z8u36aqnecndtzlwgf7s" - }, - { - "name": "grp_3:cheqd1va3slq5k97a008gxncn9eavvf4mt84xcmej65l" - }, - { - "name": "grp_3:cheqd1vex9phvwcwr785k2jal7tv2pklysqx94kzay8a" - }, - { - "name": "grp_3:cheqd1vh0rar0jgyx8xeett26nv77jnvxr99k8q8qa6q" - }, - { - "name": "grp_3:cheqd1vj9qtf9fy00q8xnwxfwsswtr24ajl8a592hyp8" - }, - { - "name": "grp_3:cheqd1vl23k3x73wzq4t7x3sh54n5f40kpse32pleka6" - }, - { - "name": "grp_3:cheqd1vlu2azc2fs9p5zffyeefth0l79eygwj8vmt2r6" - }, - { - "name": "grp_3:cheqd1vusv4s233mw867ua63m7upjxzgcyqhwhumqe6d" - }, - { - "name": "grp_3:cheqd1vxfyppcnu6lz72mdc706mlv7kx8434jaknhgt6" - }, - { - "name": "grp_3:cheqd1w5xsmev2zv59yhtk5nsl6xrdwpu588k5su6xs0" - }, - { - "name": "grp_3:cheqd1wsf8ns8gmcq76nes484pt46c9ru9c9mdmwm7sc" - }, - { - "name": "grp_3:cheqd1wwwncrnde9myewpn8j00xeyt4hfs3sqmkyasqm" - }, - { - "name": "grp_3:cheqd1wznaxsqya0meesp9f8xrknnreuszfd8kup3mq5" - }, - { - "name": "grp_3:cheqd1wzyfq9ujg7vp3whn7nh7w5ner8p7ghk2pxw0g5" - }, - { - "name": "grp_3:cheqd1x7ku9c2x2j263yaj48ct9nwjark75f0hhgmpvg" - }, - { - "name": "grp_3:cheqd1x9dc5enk7f77apgscc3emep5nz0fapxm4w9r9m" - }, - { - "name": "grp_3:cheqd1xqgm3nw0uh4k4zj0u4l2ljfqslrfumff5syu49" - }, - { - "name": "grp_3:cheqd1xqxz5wneg2hqg9tugzuv8v8wgyr2pl32v2h3hl" - }, - { - "name": "grp_3:cheqd1xs7kvk7n7m6ca6yw7ysxxfulwauc2s46z2w283" - }, - { - "name": "grp_3:cheqd1xusdhhtlqk0vetgqg7nu39j9q62zq63q22k463" - }, - { - "name": "grp_3:cheqd1xwnu2mm96uexuergqk0kqt4ds84x9zsjyne49e" - }, - { - "name": "grp_3:cheqd1xyyygaxx384k07sv3rk9zj7dezuzf250pahr0w" - }, - { - "name": "grp_3:cheqd1y6ekzultqshf8xwtv46mlzfquj6ys3vmncascq" - }, - { - "name": "grp_3:cheqd1y8edvwl536rxuqdznh48mh6mg648gf9a3xx2s9" - }, - { - "name": "grp_3:cheqd1ygwl8m54cpenq7t5v5p0y9xau3gmg873fqlqag" - }, - { - "name": "grp_3:cheqd1ylfukxalf0zr2wdma0gx33e9fccha94qr59rj4" - }, - { - "name": "grp_3:cheqd1yp52vx7sjv2dhhfqe2n52ntr0av39lye9fx5gl" - }, - { - "name": "grp_3:cheqd1yqj0s0htaseqc9prkcsmucme3ef5fsyq5mkqrd" - }, - { - "name": "grp_3:cheqd1zh8g8lw0zhp0qk0fxjq9n5p2yjc4xgpca5gzds" - }, - { - "name": "grp_3:cheqd1zhqu2pzpsv0x5afmw3tm5579pqxy59msh8ytj3" - }, - { - "name": "grp_3:cheqd1zhw4qtrd04kp7zz329zjlwy0s3vjunxua6rrqd" - }, - { - "name": "grp_3:cheqd1zm2pz9gj4vsl278vydapx0x8e3fguvxalg7n98" - }, - { - "name": "grp_3:cheqd1zmxl82zfgqsc0efuyhateaen8mzp7ngkaw4qxz" - }, - { - "name": "grp_3:cheqd1zu4mm0lzhm7az4fwc6d28pkt2vvpte5ye45rgv" - }, - { - "name": "grp_3:cheqd1zym5ync279m2sxx99dhtv4s00apr08lzw2g8g8" - }, - { - "name": "grp_3:cheqd1zz0gck564g3dvaxctccc5kgxw2sgtes7p2led6" - }, - { - "name": "grp_4:delayed:cheqd10vmxqqnc9j7axh808ymyz8t8kneux5n8xmmhq9" - }, - { - "name": "grp_4:delayed:cheqd14c5zvsf74uayyp90mav49gx2aj250s8u8yv7y9" - }, - { - "name": "grp_4:delayed:cheqd15yd3fnn3ev5xeacazejgejd5jxrnpusg8mgu8h" - }, - { - "name": "grp_4:delayed:cheqd15zw9j5e3l4l3k6fex8e7hznzmf0mpz5vcwlrt3" - }, - { - "name": "grp_4:delayed:cheqd17n6mefp8vhyrtvavc9fd9hg0hx624skgdjtrpz" - }, - { - "name": "grp_4:delayed:cheqd184z8z58dmax77csg0a9tneku0sqskeqymx5hxw" - }, - { - "name": "grp_4:delayed:cheqd18jc49zfkew8aj8zmxjyrk3nr8asqu8jxg56fwz" - }, - { - "name": "grp_4:delayed:cheqd198yncktukks0aqtyk4vq0hw20v67ffr0rm8cla" - }, - { - "name": "grp_4:delayed:cheqd1a0vumnau34trnvjpnqr4ywwaqd89l4s7ytwkk5" - }, - { - "name": "grp_4:delayed:cheqd1ajefdpfng3l66gr2j3ygvh6efrr6twrd3968rk" - }, - { - "name": "grp_4:delayed:cheqd1c5wva37ufnpdypxkxeya9kawrzgay6evnra50u" - }, - { - "name": "grp_4:delayed:cheqd1cpj9vd5e3dyvyj0nds3hanz30zafakdc4h00wj" - }, - { - "name": "grp_4:delayed:cheqd1crws5sz8tdl37v40tjn0lmlu6ztjhxar9jklqz" - }, - { - "name": "grp_4:delayed:cheqd1dy69x2g8vj0sdnutr29lt2wqgxqw98pcxltuu5" - }, - { - "name": "grp_4:delayed:cheqd1eraud0nswfglksl9tl8e5dlemrq5eualjftv3z" - }, - { - "name": "grp_4:delayed:cheqd1ewhshdq9cdhxwx8vrnnffhfe8qw3szryqv4gu7" - }, - { - "name": "grp_4:delayed:cheqd1exe54lymn5mhpjs0vj6yppnecdmtfss2qcd4cq" - }, - { - "name": "grp_4:delayed:cheqd1g3379fkvfc6jgxa66dswzsmzeguryv6zkxca87" - }, - { - "name": "grp_4:delayed:cheqd1gz3suxvpqrdwntnn7r95eerwa3ura92kp4tm8c" - }, - { - "name": "grp_4:delayed:cheqd1jkzd7sn6dzn54tl0y4ztw7n68r0p4aw53f5mqx" - }, - { - "name": "grp_4:delayed:cheqd1k3vc8xplwtqpyt55ysx085lhr09xxvhuq9ddra" - }, - { - "name": "grp_4:delayed:cheqd1kaascflyltj3pp54hwarnat6kmwtw2zvllsqdc" - }, - { - "name": "grp_4:delayed:cheqd1kq5atufq4ryhdld8s7cmy2735uxhh6wdakj9cj" - }, - { - "name": "grp_4:delayed:cheqd1kysvc7hq8944jq68yjg7hqxud66r3ajx8uv5s8" - }, - { - "name": "grp_4:delayed:cheqd1la5de5x52qw8vspwj70g6c6mem02al8sf4z4sm" - }, - { - "name": "grp_4:delayed:cheqd1lp0lq9mhc22y42znga5mvjngjpdj4pgvgzs950" - }, - { - "name": "grp_4:delayed:cheqd1lvgq858vk9hs9stfkkv0dhhvnwv5eczjx4rkjw" - }, - { - "name": "grp_4:delayed:cheqd1m5r4emnpmgnl0xgptk4f4pkrqmnlrzpnhs8aqu" - }, - { - "name": "grp_4:delayed:cheqd1mn83wm6f43f9ls6lsju8w2x4z4rg4rcvmcsq7z" - }, - { - "name": "grp_4:delayed:cheqd1nn7rcyvfd3zqhumuwwj05wvdh4x28f2qs3sex5" - }, - { - "name": "grp_4:delayed:cheqd1ppuya3ezvs2jq7n2e0ygec6r3uhx0gea6hl9vp" - }, - { - "name": "grp_4:delayed:cheqd1pw0z58t0jrtexp26694ycrdgv6ycmt5fcy00vg" - }, - { - "name": "grp_4:delayed:cheqd1q4lkgp3psk2fdxt2pgr0lujyj5fk4lz2wlz8dy" - }, - { - "name": "grp_4:delayed:cheqd1q6wwcqf902ygu8z7dcyhh8ca0u6a5g4kns6he8" - }, - { - "name": "grp_4:delayed:cheqd1qxr4jdgug8dve9kq6vc6pwdhpuyvlsql85xx0z" - }, - { - "name": "grp_4:delayed:cheqd1r2y0ffh5xgkrnzelygz35mprjejaatnah9untf" - }, - { - "name": "grp_4:delayed:cheqd1s0sngupwnkfv84r079x5ct74j8nnxgjfp89p54" - }, - { - "name": "grp_4:delayed:cheqd1ssge5ykw4hvu4jy6eeen7hyxnw72mds6mzj79h" - }, - { - "name": "grp_4:delayed:cheqd1ssjkeq5k385fydn9n7qvcp74pqkxgl8r8eczfq" - }, - { - "name": "grp_4:delayed:cheqd1sueav9pmjy5k8crpg2nuakyssd86kcrrlc0qre" - }, - { - "name": "grp_4:delayed:cheqd1t62uzdkyaw27v59ufyftzccl707k8a6rt9kcgn" - }, - { - "name": "grp_4:delayed:cheqd1w53yz6ylpraykng3sw6r8lunuar7lteavel3x0" - }, - { - "name": "grp_4:delayed:cheqd1x6srspstwx3pdta64d99en9radmaztms6lj0te" - }, - { - "name": "grp_4:delayed:cheqd1y3fvu2ychq7247fkwfna0nn6exl9w6u009tnh7" - }, - { - "name": "grp_4:delayed:cheqd1z8hvr2drwgnfqudkrqhwvdv5p40h8maw7jg92g" - } - ] -} diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 4c00c400..7843d9d2 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -61,7 +61,7 @@ export class BigDipperApi { } }`; - let resp = await this.graphql_client.query<{ supply: { coins: Coin[] }[] }>(query); + let resp = await this.graphql_client.query<{ data: { supply: { coins: Coin[] }[] } }>(query); return resp.data.supply[0].coins; } diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index a14e4903..ce346564 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -7,7 +7,7 @@ export async function updateGroupBalances(group: number, event: Event) { let balances: { account: Account } [] = []; const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${group}:` }); - // const cached = await import ("../../addresses.json"); + console.log(`found ${cached.keys.length} cached accounts`) for (const key of cached.keys) { From ec730d9271c8ecf86db7dda8b3d4601c3dd66240 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Mon, 3 Oct 2022 07:33:48 -0300 Subject: [PATCH 045/224] Removed unecessary endpoint for processing balance groups --- src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 713688fa..e0b9ac81 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,8 +34,6 @@ function registerRoutes(router: Router) { router.get('/supply/circulating', circulatingSupplyHandler); router.get('/supply/staked', totalStakedCoinsHandler); router.get('/supply/total', totalSupplyHandler); - router.get('/_/grp/:grp', balanceUpdaterHandler); - // 404 for all other requests router.all('*', () => new Response('Not Found.', { status: 404 })) From f2d8a1c3edb0d8ed3fff9e62c80f67736f52f32e Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Mon, 3 Oct 2022 07:35:05 -0300 Subject: [PATCH 046/224] Added check for null account on balance endpoint --- src/handlers/totalBalance.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 31ab6717..e10f7bb0 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -28,7 +28,10 @@ export async function handler(request: Request): Promise { let bd_api = new BigDipperApi(gql_client); let account = await bd_api.get_account(address); - let balance = total_balance_ncheq(account); - return new Response(ncheq_to_cheq_fixed(balance)) + if (!account) { + throw new Error("Account not found"); + } + + return new Response(ncheq_to_cheq_fixed(total_balance_ncheq(account))) } From b83bd32d9ec592457ff504334e455365b2d3e2a2 Mon Sep 17 00:00:00 2001 From: Daniel Ribeiro Date: Mon, 3 Oct 2022 14:57:33 -0300 Subject: [PATCH 047/224] Added updated at to KV record --- src/handlers/circulatingSupply.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index a7800eda..8830e8cd 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -24,7 +24,7 @@ async function get_circulating_supply(): Promise { if (data !== null) { if (data.totalBalance === undefined) { const balance = total_balance_ncheq(JSON.parse(data) as Account) - data = JSON.stringify({ totalBalance: balance }) + data = JSON.stringify({ totalBalance: balance, updatedAt: Date.now() }) console.log(`updating bad cache entry: ${JSON.stringify(data)} totalBalance=${data.totalBalance} data=${JSON.stringify(data)}`) await CIRCULATING_SUPPLY_WATCHLIST.put(r.name, data) } From ed201d1591252582b39d19957479e81a67d8e1be Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 12 Oct 2022 23:56:34 +0100 Subject: [PATCH 048/224] Squashed commit of the following: commit 2e5ba95cb22451f8e741be09a519e683134f7d4f Author: Ankur Banerjee Date: Wed Oct 12 22:51:19 2022 +0100 ci: Decouple production deploy from staging commit a9fc8c27e2cd13fa91754838cb12e613046044d3 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed Oct 12 09:20:00 2022 +0100 chore(deps-dev): Bump wrangler from 2.1.9 to 2.1.11 (#34) Bumps [wrangler](https://github.com/cloudflare/wrangler2/tree/HEAD/packages/wrangler) from 2.1.9 to 2.1.11. - [Release notes](https://github.com/cloudflare/wrangler2/releases) - [Changelog](https://github.com/cloudflare/wrangler2/blob/main/packages/wrangler/CHANGELOG.md) - [Commits](https://github.com/cloudflare/wrangler2/commits/wrangler@2.1.11/packages/wrangler) --- updated-dependencies: - dependency-name: wrangler dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> commit 1e0b027ef177cec5f89f228e992abfa36e449a8b Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed Oct 12 08:56:16 2022 +0100 chore(deps): Bump itty-router from 2.6.5 to 2.6.6 (#30) Bumps [itty-router](https://github.com/kwhitley/itty-router) from 2.6.5 to 2.6.6. - [Release notes](https://github.com/kwhitley/itty-router/releases) - [Changelog](https://github.com/kwhitley/itty-router/blob/v2.x/CHANGELOG.md) - [Commits](https://github.com/kwhitley/itty-router/commits/v2.6.6) --- updated-dependencies: - dependency-name: itty-router dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> commit f2a10c3224af2f4dd088e83bdd84e6daf9be3736 Author: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed Oct 12 03:26:34 2022 +0300 feat: Design and implement monitoring for price discrepancies across exchanges [DEV-1300] (#29) * Price discrepancy API * Use direct value return * Function checks * fix: use forEach instead of for loop in Prices API * Update .gitignore * Dynamic CoinGecko marketplaces * chore: switch to env vars and minor changes * chore: add readme for prices endpoint * chore: change arbitrage threshold * Set GET handler for CoinGecko API * Correct spelling mistake * Rename to arbitrage * Rename Arbitrage import * Set CoinGecko ID in Wrangler * Set CoinGecko ID in binding * Update wrangler.toml * Remove CHEQ_COIN_ID from constants * Removed constants file * Update currency.ts * Rename to CHEQ_COIN_ID to COINGECKO_ID * Install CoinGecko API package * feat: move market monitor logic to DO functions * chore: update readme and deps * Updated documentation for arbitrage * Fix typo in marketMonitoring types * chore: switch to the deployed version of market monitoring API * chore: restructure response and add /all endpoint for data dump * fix: market monitoring endpoint * chore: change to camelCase * chore: remove access token for market API * chore: add WEBHOOK_URL secret to deploy workflow * Add webhook URL secret to staging deploy * Rename marketing monitoring API constants * Rename coinPriceHandler * Make constants match * Remove concurrency guard * Change staging cron schedule Signed-off-by: jay-dee7 Co-authored-by: nikola-cheqd Co-authored-by: jay-dee7 Co-authored-by: Ankur Banerjee --- .github/workflows/deploy.yml | 16 ++++-- .github/workflows/dispatch.yml | 3 - README.md | 42 ++++++++++---- package-lock.json | 68 ++++++++++++----------- package.json | 8 +-- src/api/marketMonitorApi.ts | 15 +++++ src/bindings.d.ts | 4 +- src/handlers/allArbitrageOpportunities.ts | 16 ++++++ src/handlers/arbitrageOpportunities.ts | 30 ++++++++++ src/handlers/webhookTriggers.ts | 31 +++++++++++ src/index.ts | 36 ++++++------ src/types/marketMonitor.ts | 22 ++++++++ wrangler.toml | 19 ++++++- 13 files changed, 239 insertions(+), 71 deletions(-) create mode 100644 src/api/marketMonitorApi.ts create mode 100644 src/handlers/allArbitrageOpportunities.ts create mode 100644 src/handlers/arbitrageOpportunities.ts create mode 100644 src/handlers/webhookTriggers.ts create mode 100644 src/types/marketMonitor.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 71628f4a..05fc6c29 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,6 +16,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + persist-credentials: false - uses: actions/setup-node@v3 with: @@ -31,14 +33,15 @@ jobs: wranglerVersion: '2.0.29' preCommands: npm ci command: publish --env staging + secrets: | + WEBHOOK_URL + env: + WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} production-deploy: name: "Cloudflare - Production" - needs: staging-deploy - if: ${{ success() && ( github.ref_name == 'main' ) }} + if: ${{ github.ref_name == 'main' }} runs-on: ubuntu-latest - permissions: - security-events: write environment: name: production url: https://data-api.cheqd.io/ @@ -47,6 +50,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + persist-credentials: false - uses: actions/setup-node@v3 with: @@ -62,3 +66,7 @@ jobs: wranglerVersion: '2.0.29' preCommands: npm ci command: publish + secrets: | + WEBHOOK_URL + env: + WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 55d83f8f..3c08a54d 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -14,7 +14,4 @@ jobs: name: "Deploy" needs: call-lint uses: ./.github/workflows/deploy.yml - concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false secrets: inherit diff --git a/README.md b/README.md index b7b8a619..45704960 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ This collection of custom APIs can be deployed as a [Cloudflare Worker](https:// #### Response -*Just* total supply of tokens, in main token denomination (CHEQ instead of `ncheq` in our case) +_Just_ total supply of tokens, in main token denomination (CHEQ instead of `ncheq` in our case) #### Rationale @@ -36,20 +36,20 @@ While this figure is available from Cosmos SDK's built-in [`/cosmos/bank/v1beta1 #### Response -Circulating token supply, in main token denomination (CHEQ instead of *ncheq* in our case) +Circulating token supply, in main token denomination (CHEQ instead of _ncheq_ in our case) #### Rationale Cryptocurrency tracking websites such as [CoinMarketCap](https://coinmarketcap.com/currencies/cheqd/) and [CoinGecko](https://www.coingecko.com/en/coins/cheqd-network) require an API endpoint for reporting the circulating supply of tokens in the main/primary token denomination. -This figure is *not* available from any Cosmos SDK API, because the [criteria for determining circulating vs "non-circulating" accounts is defined by CoinMarketCap](https://support.coinmarketcap.com/hc/en-us/articles/360043396252-Supply-Circulating-Total-Max-). +This figure is _not_ available from any Cosmos SDK API, because the [criteria for determining circulating vs "non-circulating" accounts is defined by CoinMarketCap](https://support.coinmarketcap.com/hc/en-us/articles/360043396252-Supply-Circulating-Total-Max-). This API calculates the circulating supply by **subtracting** the account balances of a defined list of wallet addresses ("circulating supply watchlist"). Different types of accounts defined in the watchlist are handled as follows: 1. **Base accounts and Continuous Vesting accounts**: These will always have an entry in BigDipper block explorer, since these accounts have transactions that trigger indexing. -2. **Delayed Vesting accounts**: These accounts present a complex scenario since BigDipper does *not* index all delayed vesting accounts by default. - 1. **If there have been ANY transactions involving the delayed vesting account**: Delayed vesting accounts can still stake their original vesting allowance, or the account holder may have transferred additional funds into the account. In this scenario, the account *will* be indexed by BigDipper and the account balance can be fetched via the GraphQL API. - 2. **If there have been NO transactions involving the delayed vesting account**: Delayed vesting accounts with no other transactions beyond the original creation are *not* indexed by BigDipper. Balances for these accounts are fetched using the standard Cosmos SDK `/cosmos/bank/v1beta1/balances/
` REST API endpoint. +2. **Delayed Vesting accounts**: These accounts present a complex scenario since BigDipper does _not_ index all delayed vesting accounts by default. + 1. **If there have been ANY transactions involving the delayed vesting account**: Delayed vesting accounts can still stake their original vesting allowance, or the account holder may have transferred additional funds into the account. In this scenario, the account _will_ be indexed by BigDipper and the account balance can be fetched via the GraphQL API. + 2. **If there have been NO transactions involving the delayed vesting account**: Delayed vesting accounts with no other transactions beyond the original creation are _not_ indexed by BigDipper. Balances for these accounts are fetched using the standard Cosmos SDK `/cosmos/bank/v1beta1/balances/
` REST API endpoint. ### đŸĨŠ Total staked supply @@ -135,7 +135,7 @@ Tokens in continuous/delayed vesting accounts that can be converted to liquid ba Tokens in [continuous or delayed vesting accounts](https://docs.cosmos.network/v0.45/modules/auth/05_vesting.html#vesting-account-types) that can be converted to liquid balances. This is calculated as the sum of the following figures: -1. "Delegated free" balance (from the `/cosmos/auth/v1beta1/accounts/
` REST API) *or* vested balance, whichever is higher +1. "Delegated free" balance (from the `/cosmos/auth/v1beta1/accounts/
` REST API) _or_ vested balance, whichever is higher 2. "Available" balance (if applicable) 3. "Reward" balance (if applicable) @@ -153,6 +153,28 @@ Total account balance for specified account, in CHEQ. The standard Cosmos SDK REST API for account balances returns JSON with the account balances along with its denomination, usually the lowest denomination. This is hard to parse in applications such as Google Sheets (e.g., to monitor the account balance by fetching a response from a REST API directly in Google Sheets). This API returns a plain number that can be directly plugged into such applications, without having to parse JSON. +### 🚨 Arbitrage + +#### Endpoint + +[`data-api.cheqd.io/arbitrage/`](https://data-api.cheqd.io/arbitrage) + +#### Response + +Returns current price of CHEQ token among different markets along with an evaluation of whether they are at risk of arbitrage opportunities. + +#### Rationale + +The CHEQ token trades on multiple markets/exchanges (e.g., [Osmosis](https://app.osmosis.zone), [Gate.io](https://www.gate.io/trade/CHEQ_USDT), [BitMart](https://www.bitmart.com/trade/en?layout=basic&symbol=CHEQ_USDT), [LBank](https://www.lbank.info/exchange/cheq/usdt), [Uniswap](https://app.uniswap.org/#/swap?inputCurrency=0x70edf1c215d0ce69e7f16fd4e6276ba0d99d4de7&outputCurrency=0xdac17f958d2ee523a2206206994597c13d831ec7&chain=mainnet)). This is typically established as CHEQ along with another token pair or currency. + +Fluctuations in the exchange rate between CHEQ and other tokens pairs can give rise to opportunities for arbitrage. Having a significant market arbitrage among different exchanges creates a [market inefficiencies](https://www.investopedia.com/terms/i/inefficientmarket.asp). Extreme market inefficiencies result [market failure](https://www.investopedia.com/terms/m/marketfailure.asp) and [deadweight loss](https://www.investopedia.com/terms/d/deadweightloss.asp). + +Having monitoring capabilities for arbitrage gives opportunities for the cheqd community to rectify potential liquidity issues and aware of exchange rate movements. + +#### Alerting via Zapier + +To alert a significant market arbitrages for CHEQ listings on different exchanges, we pull latest markets data from the [CoinGecko API for cheqd's ticker page](https://www.coingecko.com/en/coins/cheqd-network) via our Market Monitoring API [Monitor Markets API](https://github.com/cheqd/market-monitoring). If an arbitrage threshold is exceeded, a webhook trigger is sent to [Zapier](https://zapier.com/) for alerting via different channels (such as Slack). + ## 🧑‍đŸ’ģ🛠 Developer Guide ### Architecture @@ -179,7 +201,7 @@ While our deployment uses Cloudflare Wrangler, the application itself could be m Wrangler CLI uses [`wrangler.toml` for configuring](https://developers.cloudflare.com/workers/wrangler/configuration/) the application. If you're using this for your own purposes, you will need to replace values for `account_id`, [Cloudflare KV](https://developers.cloudflare.com/workers/learning/how-kv-works/) bindings, `route`, etc. for the application to work correctly along with your own [Cloudflare API tokens](https://developers.cloudflare.com/api/tokens/create). -For the circulating supply API endpoint, Cloudflare Workers will expect to find a Cloudflare KV namespace called `CIRCULATING_SUPPLY_WATCHLIST` with a list of addresses in the `key`. The application *only* uses the key, so value can be anything. +For the circulating supply API endpoint, Cloudflare Workers will expect to find a Cloudflare KV namespace called `CIRCULATING_SUPPLY_WATCHLIST` with a list of addresses in the `key`. The application _only_ uses the key, so value can be anything. Delayed vesting accounts that have never been involved in a transaction (as described above) should be prefixed with a `delayed:` prefix in the JSON file. Cloudflare allows [filtering KV pair `key`s by prefixes](https://developers.cloudflare.com/workers/runtime-apis/kv/#more-detail) when using a list operation. @@ -193,7 +215,7 @@ Delayed vesting accounts that have never been involved in a transaction (as desc { "key": "delayed:cheqd1...xxx", // This is a delayed account that won't be indexed by BigDipper "value": "26-May-2022" - }, + } ] ``` @@ -231,7 +253,7 @@ Wrangler CLI also allows a degree of local development by running the web framew wrangler dev --local ``` -If you want *completely* standalone local development, this can achieved using an emulator framework like [Miniflare](https://miniflare.dev/). +If you want _completely_ standalone local development, this can achieved using an emulator framework like [Miniflare](https://miniflare.dev/). ### Deploy diff --git a/package-lock.json b/package-lock.json index 101583f9..fc94a070 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,21 +1,21 @@ { "name": "data-api", - "version": "1.0.3", + "version": "1.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "data-api", - "version": "1.0.3", + "version": "1.1.0", "license": "Apache-2.0", "dependencies": { - "itty-router": "^2.6.5" + "itty-router": "^2.6.6" }, "devDependencies": { "@cloudflare/workers-types": "^3.16.0", "@types/node": "^17.0.45", - "typescript": "^4.8.3", - "wrangler": "^2.1.6" + "typescript": "^4.8.4", + "wrangler": "^2.1.11" } }, "node_modules/@cloudflare/kv-asset-handler": { @@ -997,9 +997,9 @@ "dev": true }, "node_modules/itty-router": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/itty-router/-/itty-router-2.6.5.tgz", - "integrity": "sha512-EzfbnYF/ksgpoKgsJuk8Sknk6QDC8hd2HeOCFmDOVT+9mmk5nOfgCTglpILBOj4zX3e1RB8r4+2js6i3Zx/KQw==" + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/itty-router/-/itty-router-2.6.6.tgz", + "integrity": "sha512-hIPHtXGymCX7Lzb2I4G6JgZFE4QEEQwst9GORK7sMYUpJvLfy4yZJr95r04e8DzoAnj6HcxM2m4TbK+juu+18g==" }, "node_modules/kleur": { "version": "4.1.5", @@ -1426,9 +1426,9 @@ } }, "node_modules/typescript": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.3.tgz", - "integrity": "sha512-goMHfm00nWPa8UvR/CPSvykqf6dVV8x/dp0c5mFTMTIu0u0FlGWRioyy7Nn0PGAdHxpJZnuO/ut+PpQ8UiHAig==", + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -1481,18 +1481,21 @@ } }, "node_modules/wrangler": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.6.tgz", - "integrity": "sha512-gwIdA5UPNA/u6U2vCVit3t75ldWmBiN11OQOK48G0u1xN/gdH/ktHWR/C8blDXTAM4c53mLJJw5u5X55ZR1LLw==", + "version": "2.1.11", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.11.tgz", + "integrity": "sha512-zXydDzU+KKOwYDD9IX+XdSZMFEPWTghzTN/CiZc+pxHGIjTuQBtbk97trY3i9YKeih/QOSlo+H7Clfoq+6rZLw==", "dev": true, "dependencies": { "@cloudflare/kv-asset-handler": "^0.2.0", "@esbuild-plugins/node-globals-polyfill": "^0.1.1", "@esbuild-plugins/node-modules-polyfill": "^0.1.4", + "@miniflare/core": "2.9.0", + "@miniflare/d1": "2.9.0", + "@miniflare/durable-objects": "2.9.0", "blake3-wasm": "^2.1.5", "chokidar": "^3.5.3", "esbuild": "0.14.51", - "miniflare": "^2.8.1", + "miniflare": "2.9.0", "nanoid": "^3.3.3", "path-to-regexp": "^6.2.0", "selfsigned": "^2.0.1", @@ -1511,9 +1514,9 @@ } }, "node_modules/ws": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.1.tgz", - "integrity": "sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA==", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.9.0.tgz", + "integrity": "sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg==", "dev": true, "engines": { "node": ">=10.0.0" @@ -2206,9 +2209,9 @@ "dev": true }, "itty-router": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/itty-router/-/itty-router-2.6.5.tgz", - "integrity": "sha512-EzfbnYF/ksgpoKgsJuk8Sknk6QDC8hd2HeOCFmDOVT+9mmk5nOfgCTglpILBOj4zX3e1RB8r4+2js6i3Zx/KQw==" + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/itty-router/-/itty-router-2.6.6.tgz", + "integrity": "sha512-hIPHtXGymCX7Lzb2I4G6JgZFE4QEEQwst9GORK7sMYUpJvLfy4yZJr95r04e8DzoAnj6HcxM2m4TbK+juu+18g==" }, "kleur": { "version": "4.1.5", @@ -2514,9 +2517,9 @@ } }, "typescript": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.3.tgz", - "integrity": "sha512-goMHfm00nWPa8UvR/CPSvykqf6dVV8x/dp0c5mFTMTIu0u0FlGWRioyy7Nn0PGAdHxpJZnuO/ut+PpQ8UiHAig==", + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", "dev": true }, "undici": { @@ -2550,19 +2553,22 @@ } }, "wrangler": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.6.tgz", - "integrity": "sha512-gwIdA5UPNA/u6U2vCVit3t75ldWmBiN11OQOK48G0u1xN/gdH/ktHWR/C8blDXTAM4c53mLJJw5u5X55ZR1LLw==", + "version": "2.1.11", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.11.tgz", + "integrity": "sha512-zXydDzU+KKOwYDD9IX+XdSZMFEPWTghzTN/CiZc+pxHGIjTuQBtbk97trY3i9YKeih/QOSlo+H7Clfoq+6rZLw==", "dev": true, "requires": { "@cloudflare/kv-asset-handler": "^0.2.0", "@esbuild-plugins/node-globals-polyfill": "^0.1.1", "@esbuild-plugins/node-modules-polyfill": "^0.1.4", + "@miniflare/core": "2.9.0", + "@miniflare/d1": "2.9.0", + "@miniflare/durable-objects": "2.9.0", "blake3-wasm": "^2.1.5", "chokidar": "^3.5.3", "esbuild": "0.14.51", "fsevents": "~2.3.2", - "miniflare": "^2.8.1", + "miniflare": "2.9.0", "nanoid": "^3.3.3", "path-to-regexp": "^6.2.0", "selfsigned": "^2.0.1", @@ -2571,9 +2577,9 @@ } }, "ws": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.1.tgz", - "integrity": "sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA==", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.9.0.tgz", + "integrity": "sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg==", "dev": true, "requires": {} }, diff --git a/package.json b/package.json index 41c33a0f..6199e419 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "data-api", - "version": "1.0.3", + "version": "1.1.0", "license": "Apache-2.0", "source": "src/index.ts", "author": "Cheqd Foundation Limited (https://github.com/cheqd)", @@ -14,13 +14,13 @@ "publish": "npx wrangler publish" }, "dependencies": { - "itty-router": "^2.6.5" + "itty-router": "^2.6.6" }, "devDependencies": { "@cloudflare/workers-types": "^3.16.0", "@types/node": "^17.0.45", - "typescript": "^4.8.3", - "wrangler": "^2.1.6" + "typescript": "^4.8.4", + "wrangler": "^2.1.11" }, "private": true } diff --git a/src/api/marketMonitorApi.ts b/src/api/marketMonitorApi.ts new file mode 100644 index 00000000..28ebcc49 --- /dev/null +++ b/src/api/marketMonitorApi.ts @@ -0,0 +1,15 @@ +import { MarketMonitorData } from '../types/marketMonitor'; +export class MarketMonitorApi { + constructor(public readonly base_market_monitor_api_url: string) {} + + async get_market_monitor_data(): Promise { + const requestOptions = { + method: 'GET', + }; + const response = await fetch( + `${this.base_market_monitor_api_url}`, + requestOptions + ); + return (await response.json()) as MarketMonitorData; + } +} diff --git a/src/bindings.d.ts b/src/bindings.d.ts index b68f259a..32c54676 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -3,6 +3,8 @@ declare global { const REST_API: string; const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; + const MARKET_MONITORING_API: string; + const WEBHOOK_URL: string; } -export {} +export {}; diff --git a/src/handlers/allArbitrageOpportunities.ts b/src/handlers/allArbitrageOpportunities.ts new file mode 100644 index 00000000..76badc5b --- /dev/null +++ b/src/handlers/allArbitrageOpportunities.ts @@ -0,0 +1,16 @@ +import { MarketMonitorApi } from "../api/marketMonitorApi"; + +export async function fetchPrices() { + let market_monitor_api = new MarketMonitorApi( + `${MARKET_MONITORING_API}` + ); + return await market_monitor_api.get_market_monitor_data(); +} +export async function handler(request: Request): Promise { + const payload = await fetchPrices(); + return new Response(JSON.stringify(payload, null, 2), { + headers: { + "content-type": "application/json;charset=UTF-8", + }, + }); +} diff --git a/src/handlers/arbitrageOpportunities.ts b/src/handlers/arbitrageOpportunities.ts new file mode 100644 index 00000000..2f89204b --- /dev/null +++ b/src/handlers/arbitrageOpportunities.ts @@ -0,0 +1,30 @@ +import { MarketMonitorApi } from '../api/marketMonitorApi'; +import { ArbitrageOpportunity } from '../types/marketMonitor'; + +async function fetchPrices() { + let market_monitor_api = new MarketMonitorApi( + `${MARKET_MONITORING_API}` + ); + return await market_monitor_api.get_market_monitor_data(); +} + +export async function filterArbitrageOpportunities(): Promise< + ArbitrageOpportunity[] +> { + const payload = await fetchPrices(); + const arbitrage_opportunities = []; + for (let i = 0; i < payload.arbitrageOpportunities.length; i++) { + if (payload.arbitrageOpportunities[i].arbitragePossible) { + arbitrage_opportunities.push(payload.arbitrageOpportunities[i]); + } + } + return arbitrage_opportunities; +} +export async function handler(request: Request): Promise { + const arbitrage_opportunities = await filterArbitrageOpportunities(); + return new Response(JSON.stringify(arbitrage_opportunities, null, 2), { + headers: { + 'content-type': 'application/json;charset=UTF-8', + }, + }); +} diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts new file mode 100644 index 00000000..f674601e --- /dev/null +++ b/src/handlers/webhookTriggers.ts @@ -0,0 +1,31 @@ +import { filterArbitrageOpportunities } from './arbitrageOpportunities'; + +export async function webhookTriggers(event: Event) { + console.log('Triggering webhook...'); + await sendPriceDiscrepancies(); +} + +export async function sendPriceDiscrepancies() { + console.log('Sending price discrepancies...'); + + const arbitrageOpportunities = await filterArbitrageOpportunities(); + const hasArbitrageOpportunities = arbitrageOpportunities.length > 0; + if (hasArbitrageOpportunities) { + console.log('Arbitrage opportunities...'); + try { + const init = { + body: JSON.stringify({ + arbitrage_opportunities: arbitrageOpportunities, + }), + method: 'POST', + headers: { + 'content-type': 'application/json;charset=UTF-8', + }, + }; + + await fetch(WEBHOOK_URL, init); + } catch (err: any) { + console.log(err); + } + } +} diff --git a/src/index.ts b/src/index.ts index e0b9ac81..1f6a6a90 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,30 +1,33 @@ -import { IHTTPMethods, Request, Router } from 'itty-router' -import { handler as totalSupplyHandler } from "./handlers/totalSupply"; -import { handler as totalBalanceHandler } from "./handlers/totalBalance"; -import { handler as circulatingSupplyHandler } from "./handlers/circulatingSupply"; -import { handler as liquidBalanceHandler } from "./handlers/liquidBalance"; -import { handler as vestingBalanceHandler } from "./handlers/vestingBalance"; -import { handler as vestedBalanceHandler } from "./handlers/vestedBalance"; +import { Router, Request, IHTTPMethods } from 'itty-router'; +import { handler as totalSupplyHandler } from './handlers/totalSupply'; +import { handler as totalBalanceHandler } from './handlers/totalBalance'; +import { handler as circulatingSupplyHandler } from './handlers/circulatingSupply'; +import { handler as liquidBalanceHandler } from './handlers/liquidBalance'; +import { handler as vestingBalanceHandler } from './handlers/vestingBalance'; +import { handler as vestedBalanceHandler } from './handlers/vestedBalance'; import { handler as delegatorCountHandler } from './handlers/delegatorCount'; import { handler as totalDelegatorsHandler } from './handlers/totalDelegators'; -import { handler as totalStakedCoinsHandler } from "./handlers/totalStakedCoins"; +import { handler as totalStakedCoinsHandler } from './handlers/totalStakedCoins'; +import { handler as allArbitrageOpportunitiesHandler } from './handlers/allArbitrageOpportunities'; +import { handler as arbitrageOpportunitiesHandler } from './handlers/arbitrageOpportunities'; +import { webhookTriggers } from './handlers/webhookTriggers'; import { handler as balanceUpdaterHandler } from "./handlers/balanceGroup"; -import { updateGroupBalances } from "./helpers/balanceGroup"; addEventListener('scheduled', (event: any) => { - console.log(`triggering scheduled account balance update`) - - event.waitUntil(updateGroupBalances(getRandomGroup(), event)); -}) + event.waitUntil(webhookTriggers(event)); + // event.waitUntil(balanceUpdaterHandler(getRandomGroup(event))); +}); addEventListener('fetch', (event: FetchEvent) => { - const router = Router() - registerRoutes(router); - event.respondWith(router.handle(event.request).catch(handleError)) + const router = Router() + registerRoutes(router); + event.respondWith(router.handle(event.request).catch(handleError)) }) function registerRoutes(router: Router) { router.get('/', totalSupplyHandler); + router.get('/arbitrage', arbitrageOpportunitiesHandler); + router.get('/arbitrage/all', allArbitrageOpportunitiesHandler); router.get('/balances/liquid/:address', liquidBalanceHandler); router.get('/balances/total/:address', totalBalanceHandler); router.get('/balances/vested/:address', vestedBalanceHandler); @@ -61,6 +64,5 @@ function getRandomGroup(): number { if (hour >= 18 && hour < 24) { return 2; } - throw new Error("invalid hour for group"); } diff --git a/src/types/marketMonitor.ts b/src/types/marketMonitor.ts new file mode 100644 index 00000000..d327789f --- /dev/null +++ b/src/types/marketMonitor.ts @@ -0,0 +1,22 @@ +export interface Price { + price: number; + coinPair: string; + market: string; +} + +export interface ArbitrageOpportunity { + marketPairId: string; + marketName1: string; + coinPair1: string; + coinPrice1: number; + marketName2: string; + coinPair2: string; + coinPrice2: number; + percentageDelta: number; + arbitragePossible: boolean; +} + +export interface MarketMonitorData { + prices: Price[]; + arbitrageOpportunities: ArbitrageOpportunity[]; +} diff --git a/wrangler.toml b/wrangler.toml index c26ec2f5..6266482f 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -45,7 +45,16 @@ TOKEN_EXPONENT = "9" REST_API = "https://api.cheqd.net" # GraphQL API endpoint for target network. Must be sourced from a BigDipper instance. GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" +# Moniter market API base url +MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" +# The necessary secrets are: +# - WEBHOOK_URL +# Run `echo | wrangler secret put ` for each of these + + +[triggers] +crons = ["0 * * * *"] ############################################################### ### SECTION 3: Local Development ### @@ -78,7 +87,12 @@ route = { pattern = "data-api-staging.cheqd.io/*", zone_id = "afe3b66243382f2714 # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` -vars = { TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" } +vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net"} + + +# The necessary secrets are: +# - WEBHOOK_URL +# Run `echo | wrangler secret put ` for each of these # KV Namespaces accessible from the Worker # Details: https://developers.cloudflare.com/workers/learning/how-kv-works @@ -88,6 +102,9 @@ binding = "CIRCULATING_SUPPLY_WATCHLIST" id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" preview_id = "1e1032cbf6854d88b12317da8a792928" +# Cron triggers for staging worker +[env.staging.triggers] +crons = ["0 9 * * *"] [triggers] From 519f9d2b725a91b26afdeb62a86b6e609584a75e Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 12 Oct 2022 23:58:21 +0100 Subject: [PATCH 049/224] Update wrangler.toml --- wrangler.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index 6266482f..aef77c7e 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -14,7 +14,7 @@ compatibility_date = "2022-05-24" # Usage model for the Worker # Details: https://developers.cloudflare.com/workers/platform/limits -usage_model = "unbound" +usage_model = "bundled" ############################################################### @@ -54,7 +54,7 @@ MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" [triggers] -crons = ["0 * * * *"] +crons = ["0 0/2 * * *"] ############################################################### ### SECTION 3: Local Development ### From 64c3ca7d23536c23fbc0581029da24c20e1bdaaf Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 13 Oct 2022 00:04:01 +0100 Subject: [PATCH 050/224] Fixed broken trigger --- wrangler.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index aef77c7e..c50d0643 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -52,7 +52,6 @@ MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" # - WEBHOOK_URL # Run `echo | wrangler secret put ` for each of these - [triggers] crons = ["0 0/2 * * *"] @@ -106,9 +105,6 @@ preview_id = "1e1032cbf6854d88b12317da8a792928" [env.staging.triggers] crons = ["0 9 * * *"] - -[triggers] -crons = ["0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### ############################################################### From a13332c99c2b84aeb3f44f89675956add30468df Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 13 Oct 2022 12:34:32 +0300 Subject: [PATCH 051/224] fix total delegations count per validatior --- src/api/bigDipperApi.ts | 154 +++++++++++++++++++++------------------- src/types/node.ts | 82 ++++++++++----------- 2 files changed, 123 insertions(+), 113 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 7843d9d2..10fc7295 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,13 +1,16 @@ -import { GraphQLClient } from "../helpers/graphql"; -import { Coin, ValidatorAggregateCountResponse, ValidatorDetailResponse } from "../types/node"; -import { Account } from "../types/bigDipper"; +import { GraphQLClient } from '../helpers/graphql'; +import { + Coin, + ValidatorDelegationsCountResponse, + ValidatorDetailResponse, +} from '../types/node'; +import { Account } from '../types/bigDipper'; export class BigDipperApi { - constructor(public readonly graphql_client: GraphQLClient) { - } + constructor(public readonly graphql_client: GraphQLClient) {} - async get_account(address: string): Promise { - let query = `query Account($address: String!, $where: vesting_account_bool_exp) { + async get_account(address: string): Promise { + let query = `query Account($address: String!, $where: vesting_account_bool_exp) { accountBalance: action_account_balance(address: $address) { coins } @@ -30,67 +33,68 @@ export class BigDipperApi { start_time end_time } - }` - - let params = { - address: address, - where: { - address: { - _eq: address - } - } - } - - try { - let resp = await this.graphql_client.query<{ - data: any, errors: any - }>(query, params); + }`; - return resp.data as Account - } catch (e: any) { - console.error(new Map(e)) - return null; - } + let params = { + address: address, + where: { + address: { + _eq: address, + }, + }, + }; + + try { + let resp = await this.graphql_client.query<{ + data: any; + errors: any; + }>(query, params); + + return resp.data as Account; + } catch (e: any) { + console.error(new Map(e)); + return null; } + } - async get_total_supply(): Promise { - let query = `query Supply { + async get_total_supply(): Promise { + let query = `query Supply { supply(order_by: {height:desc} limit: 1) { coins height } }`; - let resp = await this.graphql_client.query<{ data: { supply: { coins: Coin[] }[] } }>(query); + let resp = await this.graphql_client.query<{ + data: { supply: { coins: Coin[] }[] }; + }>(query); - return resp.data.supply[0].coins; - } + return resp.data.supply[0].coins; + } - get_delegator_count_for_validator = async (address: string): Promise => { - let query = `query ValidatorDetails($address: String) { - validator(where: {validator_info: {operator_address: {_eq: $address}}}) { - delegations_aggregate { - aggregate { - count - } - } - } - }` - - const params = { - address: address, + get_delegator_count_for_validator = async ( + address: string + ): Promise => { + let query = `query ValidatorDelegations($address: String!, $pagination: Boolean! = true) { + delegations: action_validator_delegations(address: $address, count_total: $pagination) { + pagination } + } + `; - const resp = await this.graphql_client.query(query, params); - if (!resp.validator || !resp.validator.length) { - return 0; - } + const params = { + address: address, + }; - return resp.validator[0].delegations_aggregate.aggregate.count; - } + const resp = await this.graphql_client.query<{ + data: ValidatorDelegationsCountResponse; + }>(query, params); - get_total_delegator_count = async (): Promise => { - const query = `query ValidatorDetails { + return resp.data.delegations.pagination.total; + }; + + get_total_delegator_count = async (): Promise => { + const query = `query ValidatorDetails { validator { validatorStatuses: validator_statuses(order_by: {height: desc}, limit: 1) { jailed @@ -99,29 +103,33 @@ export class BigDipperApi { delegatorAddress: delegator_address } } - }` - - const resp = await this.graphql_client.query(query); - const set = new Set(); - resp.validator.forEach((obj, i) => { - if (!obj.validatorStatuses[0]?.jailed) { - obj.delegations.forEach(delegation => { - set.add(delegation.delegatorAddress) - }) - } - }) - - return set.size - } + }`; - get_total_staked_coins = async (): Promise => { - let query = `query StakingInfo{ + const resp = await this.graphql_client.query( + query + ); + const set = new Set(); + resp.validator.forEach((obj, i) => { + if (!obj.validatorStatuses[0]?.jailed) { + obj.delegations.forEach((delegation) => { + set.add(delegation.delegatorAddress); + }); + } + }); + + return set.size; + }; + + get_total_staked_coins = async (): Promise => { + let query = `query StakingInfo{ staking_pool { bonded_tokens } - }` + }`; - const resp = await this.graphql_client.query<{ staking_pool: [ { "bonded_tokens": string } ] }>(query); - return resp.staking_pool[0].bonded_tokens; - } + const resp = await this.graphql_client.query<{ + staking_pool: [{ bonded_tokens: string }]; + }>(query); + return resp.staking_pool[0].bonded_tokens; + }; } diff --git a/src/types/node.ts b/src/types/node.ts index 385f4f3e..3ffdeda5 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -1,58 +1,60 @@ export type Account = { - '@type': string; - start_time: number; - base_vesting_account: { base_account: BaseAccount, original_vesting: Coin[], delegated_free?: Coin[], delegated_vesting?: Coin[], end_time: number }; -} + '@type': string; + start_time: number; + base_vesting_account: { + base_account: BaseAccount; + original_vesting: Coin[]; + delegated_free?: Coin[]; + delegated_vesting?: Coin[]; + end_time: number; + }; +}; export type BaseAccount = { - address: string; - pub_key: PublicKey; - account_number: string; - sequence: string; -} + address: string; + pub_key: PublicKey; + account_number: string; + sequence: string; +}; export type PublicKey = { - '@type': string; - key: string; -} + '@type': string; + key: string; +}; export class Coin { - public denom: string; - public amount: string; + public denom: string; + public amount: string; - constructor(denom: string, amount: string) { - this.denom = denom; - this.amount = amount; - } + constructor(denom: string, amount: string) { + this.denom = denom; + this.amount = amount; + } } export class Delegation { - public amount: Coin; - public delegatorAddress: string; + public amount: Coin; + public delegatorAddress: string; - constructor(amount: Coin, delegatorAddress: string) { - this.delegatorAddress = delegatorAddress; - this.amount = amount; - } + constructor(amount: Coin, delegatorAddress: string) { + this.delegatorAddress = delegatorAddress; + this.amount = amount; + } } -export interface ValidatorAggregateCountResponse { - validator: [ - { - delegations_aggregate: { - aggregate: { - count: number - } - } - } - ] +export interface ValidatorDelegationsCountResponse { + delegations: { + pagination: { + total: number; + }; + }; } export interface ValidatorDetailResponse { - validator: [ - { - validatorStatuses: [{ jailed: boolean }], - delegations: [{ delegatorAddress: string }] - } - ] + validator: [ + { + validatorStatuses: [{ jailed: boolean }]; + delegations: [{ delegatorAddress: string }]; + } + ]; } From 3113a4e848c3e7d53871b528c4756149c0dad456 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 13 Oct 2022 19:37:17 +0300 Subject: [PATCH 052/224] fix totat delegators count --- src/api/bigDipperApi.ts | 56 +++++++++++++--------- src/api/nodeApi.ts | 83 ++++++++++++++++++++------------- src/handlers/totalDelegators.ts | 14 +++--- src/types/node.ts | 16 +++++-- 4 files changed, 105 insertions(+), 64 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 10fc7295..4c0310c9 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,10 +1,12 @@ import { GraphQLClient } from '../helpers/graphql'; import { + ActiveValidatorsResponse, Coin, ValidatorDelegationsCountResponse, ValidatorDetailResponse, } from '../types/node'; import { Account } from '../types/bigDipper'; +import { NodeApi } from './nodeApi'; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} @@ -94,30 +96,40 @@ export class BigDipperApi { }; get_total_delegator_count = async (): Promise => { - const query = `query ValidatorDetails { - validator { - validatorStatuses: validator_statuses(order_by: {height: desc}, limit: 1) { - jailed - } - delegations { - delegatorAddress: delegator_address - } - } - }`; + const queryActiveValidators = `query ActiveValidators { + validator_info(distinct_on: operator_address, where: {validator: {validator_statuses: {jailed: {_eq: false}}}}) { + operator_address + } + }`; + + const data = []; + const uniques = new Set(); + + const activeValidator = await this.graphql_client.query<{ + data: ActiveValidatorsResponse; + }>(queryActiveValidators); + + for (let i = 0; i < activeValidator.data.validator_info.length; i++) { + const operator_address = + activeValidator.data.validator_info[i].operator_address; + const resp = await new NodeApi( + REST_API + ).staking_get_delegators_per_validator(operator_address); + data.push({ + validator: operator_address, + delegators: resp.delegation_responses, + }); + } - const resp = await this.graphql_client.query( - query - ); - const set = new Set(); - resp.validator.forEach((obj, i) => { - if (!obj.validatorStatuses[0]?.jailed) { - obj.delegations.forEach((delegation) => { - set.add(delegation.delegatorAddress); - }); + for (let i = 0; i < data.length; i++) { + const delegators = data[i].delegators; + for (let j = 0; j < delegators.length; j++) { + uniques.add( + `${delegators[j].delegation.delegator_address}${delegators[j].delegation.validator_address}` + ); } - }); - - return set.size; + } + return uniques.size; }; get_total_staked_coins = async (): Promise => { diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 70cc7dfe..4b6ca052 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -1,35 +1,54 @@ -import { Account, Coin } from "../types/node"; +import { Account, Coin, ValidatorDetailResponse } from '../types/node'; export class NodeApi { - - constructor(public readonly base_rest_api_url: string) { - } - - async bank_get_total_supply_ncheq(): Promise { - let resp = await fetch(`${this.base_rest_api_url}/cosmos/bank/v1beta1/supply/ncheq`); - let respJson = await resp.json() as { amount: { amount: number } }; - - return respJson.amount.amount; - } - - async auth_get_account(address: string): Promise { - let resp = await fetch(`${this.base_rest_api_url}/cosmos/auth/v1beta1/accounts/${address}`) - let respJson = await resp.json() as { account: Account }; - - return respJson.account; - } - - async bank_get_account_balances(address: string): Promise { - let resp = await fetch(`${this.base_rest_api_url}/cosmos/bank/v1beta1/balances/${address}`) - let respJson = await resp.json() as { balances: Coin[] }; - - return respJson.balances; - } - - async distribution_get_total_rewards(address: string): Promise { - let resp = await fetch(`${this.base_rest_api_url}/cosmos/distribution/v1beta1/delegators/${address}/rewards`) - let respJson = await resp.json() as { rewards: Record[], total: Coin[] }; - - return Number(respJson?.total?.[0]?.amount ?? '0'); - } + constructor(public readonly base_rest_api_url: string) {} + + async bank_get_total_supply_ncheq(): Promise { + let resp = await fetch( + `${this.base_rest_api_url}/cosmos/bank/v1beta1/supply/ncheq` + ); + let respJson = (await resp.json()) as { amount: { amount: number } }; + + return respJson.amount.amount; + } + + async auth_get_account(address: string): Promise { + let resp = await fetch( + `${this.base_rest_api_url}/cosmos/auth/v1beta1/accounts/${address}` + ); + let respJson = (await resp.json()) as { account: Account }; + + return respJson.account; + } + + async bank_get_account_balances(address: string): Promise { + let resp = await fetch( + `${this.base_rest_api_url}/cosmos/bank/v1beta1/balances/${address}` + ); + let respJson = (await resp.json()) as { balances: Coin[] }; + + return respJson.balances; + } + + async distribution_get_total_rewards(address: string): Promise { + let resp = await fetch( + `${this.base_rest_api_url}/cosmos/distribution/v1beta1/delegators/${address}/rewards` + ); + let respJson = (await resp.json()) as { + rewards: Record[]; + total: Coin[]; + }; + + return Number(respJson?.total?.[0]?.amount ?? '0'); + } + + async staking_get_delegators_per_validator( + address: string + ): Promise { + let resp = await fetch( + `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?pagination.limit=100000000000000` + ); + + return await resp.json(); + } } diff --git a/src/handlers/totalDelegators.ts b/src/handlers/totalDelegators.ts index eb14f5e7..e6dce728 100644 --- a/src/handlers/totalDelegators.ts +++ b/src/handlers/totalDelegators.ts @@ -1,11 +1,11 @@ -import { Request } from "itty-router"; -import { BigDipperApi } from "../api/bigDipperApi"; -import { GraphQLClient } from "../helpers/graphql"; +import { Request } from 'itty-router'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); - const delegators = await bd_api.get_total_delegator_count(); - return new Response(JSON.stringify(delegators)); + const delegators = await bd_api.get_total_delegator_count(); + return new Response(JSON.stringify(delegators)); } diff --git a/src/types/node.ts b/src/types/node.ts index 3ffdeda5..9ddaa8d7 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -51,10 +51,20 @@ export interface ValidatorDelegationsCountResponse { } export interface ValidatorDetailResponse { - validator: [ + delegation_responses: [ { - validatorStatuses: [{ jailed: boolean }]; - delegations: [{ delegatorAddress: string }]; + delegation: { + delegator_address: string; + validator_address: string; + }; + } + ]; +} + +export interface ActiveValidatorsResponse { + validator_info: [ + { + operator_address: string; } ]; } From 3c8b55ab186726f9fcd485dda3d8429509a5bc50 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 13 Oct 2022 19:46:58 +0300 Subject: [PATCH 053/224] fix total staked coins --- src/api/bigDipperApi.ts | 6 +++--- src/handlers/totalStakedCoins.ts | 17 +++++++++-------- src/types/node.ts | 8 ++++++++ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 4c0310c9..17bbf013 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -2,8 +2,8 @@ import { GraphQLClient } from '../helpers/graphql'; import { ActiveValidatorsResponse, Coin, + TotalStakedCoinsResponse, ValidatorDelegationsCountResponse, - ValidatorDetailResponse, } from '../types/node'; import { Account } from '../types/bigDipper'; import { NodeApi } from './nodeApi'; @@ -140,8 +140,8 @@ export class BigDipperApi { }`; const resp = await this.graphql_client.query<{ - staking_pool: [{ bonded_tokens: string }]; + data: TotalStakedCoinsResponse; }>(query); - return resp.staking_pool[0].bonded_tokens; + return resp.data.staking_pool[0].bonded_tokens; }; } diff --git a/src/handlers/totalStakedCoins.ts b/src/handlers/totalStakedCoins.ts index ccdcd109..c4ffd105 100644 --- a/src/handlers/totalStakedCoins.ts +++ b/src/handlers/totalStakedCoins.ts @@ -1,12 +1,13 @@ -import { Request } from "itty-router"; -import { BigDipperApi } from "../api/bigDipperApi"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; -import { GraphQLClient } from "../helpers/graphql"; +import { Request } from 'itty-router'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); - let total_staked_coins = await bd_api.get_total_staked_coins(); - return new Response(ncheq_to_cheq_fixed(Number(total_staked_coins))); + let total_staked_coins = await bd_api.get_total_staked_coins(); + + return new Response(ncheq_to_cheq_fixed(Number(total_staked_coins))); } diff --git a/src/types/node.ts b/src/types/node.ts index 9ddaa8d7..16faa481 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -68,3 +68,11 @@ export interface ActiveValidatorsResponse { } ]; } + +export interface TotalStakedCoinsResponse { + staking_pool: [ + { + bonded_tokens: string; + } + ]; +} From b66692c147b95afd7f5146d179e26a927be4d8b5 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 14 Oct 2022 11:14:52 +0100 Subject: [PATCH 054/224] Update wrangler.toml --- wrangler.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index 8a029098..000e6811 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -14,7 +14,7 @@ compatibility_date = "2022-05-24" # Usage model for the Worker # Details: https://developers.cloudflare.com/workers/platform/limits -usage_model = "bundled" +usage_model = "unbound" ############################################################### @@ -52,9 +52,8 @@ MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" # - WEBHOOK_URL # Run `echo | wrangler secret put ` for each of these - [triggers] -crons = ["0 * * * *"] +crons = ["0 0/2 * * *"] ############################################################### ### SECTION 3: Local Development ### From 61766fb8daa279a295f7f2e6b14a262d6291acc5 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 14 Oct 2022 14:07:24 +0300 Subject: [PATCH 055/224] fix: total balance --- src/handlers/totalBalance.ts | 81 +++++++++++++++++++++--------------- src/helpers/node.ts | 43 ++++++++++++------- src/types/bigDipper.ts | 62 +++++++++++++-------------- 3 files changed, 105 insertions(+), 81 deletions(-) diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index e10f7bb0..4be2220d 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,37 +1,50 @@ -import { Request } from "itty-router"; -import { is_delayed_vesting_account_type, validate_cheqd_address } from "../helpers/validate"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; -import { BigDipperApi } from "../api/bigDipperApi"; -import { GraphQLClient } from "../helpers/graphql"; -import { total_balance_ncheq } from "../helpers/node"; -import { NodeApi } from "../api/nodeApi"; +import { Request } from 'itty-router'; +import { + is_delayed_vesting_account_type, + validate_cheqd_address, +} from '../helpers/validate'; +import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { GraphQLClient } from '../helpers/graphql'; +import { total_balance_ncheq } from '../helpers/node'; +import { NodeApi } from '../api/nodeApi'; export async function handler(request: Request): Promise { - const address = request.params?.['address']; - - if (!address || !validate_cheqd_address(address)) { - throw new Error("No address specified or wrong address format."); - } - - let node_api = new NodeApi(REST_API); - let auth_account = await node_api.auth_get_account(address); - - if (is_delayed_vesting_account_type(auth_account?.["@type"])) { - let balance = Number(await (await node_api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0'); - let rewards = Number(await (await node_api.distribution_get_total_rewards(address)) ?? '0'); - let delegated = Number(auth_account?.base_vesting_account?.delegated_vesting?.find(d => d.denom === "ncheq")?.amount ?? '0'); - - return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); - } - - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); - - let account = await bd_api.get_account(address); - - if (!account) { - throw new Error("Account not found"); - } - - return new Response(ncheq_to_cheq_fixed(total_balance_ncheq(account))) + const address = request.params?.['address']; + + if (!address || !validate_cheqd_address(address)) { + throw new Error('No address specified or wrong address format.'); + } + + let node_api = new NodeApi(REST_API); + let auth_account = await node_api.auth_get_account(address); + + if (is_delayed_vesting_account_type(auth_account?.['@type'])) { + let balance = Number( + (await ( + await node_api.bank_get_account_balances(address) + ).find((b) => b.denom === 'ncheq')?.amount) ?? '0' + ); + let rewards = Number( + (await await node_api.distribution_get_total_rewards(address)) ?? '0' + ); + let delegated = Number( + auth_account?.base_vesting_account?.delegated_vesting?.find( + (d) => d.denom === 'ncheq' + )?.amount ?? '0' + ); + + return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); + } + + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); + + let account = await bd_api.get_account(address); + + if (!account) { + throw new Error('Account not found'); + } + + return new Response(ncheq_to_cheq_fixed(total_balance_ncheq(account))); } diff --git a/src/helpers/node.ts b/src/helpers/node.ts index f58653b4..5419edd5 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,27 +1,38 @@ -import { Account } from "../types/bigDipper"; -import { Coin } from "../types/node"; +import { Account } from '../types/bigDipper'; +import { Coin } from '../types/node'; export function total_balance_ncheq(account: Account): number { - let balance = Number(account?.accountBalance?.coins.find(c => c.denom === "ncheq")?.amount || '0'); + let balance = Number( + account?.accountBalance?.coins.find((c) => c.denom === 'ncheq')?.amount || + '0' + ); - let delegations = 0; - if (account?.delegationBalance?.coins && account?.delegationBalance?.coins.length > 0) { - delegations = Number(account?.delegationBalance?.coins[0].amount); - } + let delegations = 0; + if ( + account?.delegationBalance?.coins && + account?.delegationBalance?.coins.length > 0 + ) { + delegations = Number(account?.delegationBalance?.coins[0].amount); + } - let unbonding = 0; - if (account?.unbondingBalance?.coins && account?.unbondingBalance?.coins.length > 0) { - unbonding = Number(account?.unbondingBalance?.coins[0]?.amount); - } + let unbonding = 0; + if ( + account?.unbondingBalance?.coins && + account?.unbondingBalance?.coins.length > 0 + ) { + unbonding = Number(account?.unbondingBalance?.coins[0]?.amount); + } - let rewards = 0; - if (account?.rewardBalance?.coins && account?.rewardBalance?.coins.length > 0) { - rewards = Number(account?.rewardBalance?.coins[0]?.amount); + let rewards = 0; + if (account?.rewardBalance?.length > 0) { + for (let i = 0; i < account?.rewardBalance.length; i++) { + rewards += Number(account?.rewardBalance[i]?.coins[0].amount); } + } - return balance + delegations + unbonding + rewards; + return balance + delegations + unbonding + rewards; } export function delayed_balance_ncheq(balance: Coin[]): number { - return Number(balance.find(c => c.denom === "ncheq")?.amount || '0'); + return Number(balance.find((c) => c.denom === 'ncheq')?.amount || '0'); } diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index 0a89cb56..8c841d54 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -1,35 +1,35 @@ -import { Coin } from "./node"; +import { Coin } from './node'; export class Account { - public accountBalance: { coins: Coin[] }; - public delegationBalance: { coins: Coin[] }; - public unbondingBalance: { coins: Coin[] }; - public rewardBalance: { coins: Coin[] }; - public vesting_account: { - id: string, - type: string, - original_vesting: Coin[], - start_time: number, - end_time: number - }[]; + public accountBalance: { coins: Coin[] }; + public delegationBalance: { coins: Coin[] }; + public unbondingBalance: { coins: Coin[] }; + public rewardBalance: [{ coins: Coin[] }]; + public vesting_account: { + id: string; + type: string; + original_vesting: Coin[]; + start_time: number; + end_time: number; + }[]; - constructor( - account_balance: { coins: Coin[] }, - delegation_balance: { coins: Coin[] }, - unbonding_balance: { coins: Coin[] }, - reward_balance: { coins: Coin[] }, - vesting_account: { - id: string, - type: string, - original_vesting: Coin[], - start_time: number, - end_time: number - }[]) { - this.accountBalance = account_balance; - this.delegationBalance = delegation_balance; - this.unbondingBalance = unbonding_balance; - this.rewardBalance = reward_balance; - this.vesting_account = vesting_account; - } + constructor( + account_balance: { coins: Coin[] }, + delegation_balance: { coins: Coin[] }, + unbonding_balance: { coins: Coin[] }, + reward_balance: [{ coins: Coin[] }], + vesting_account: { + id: string; + type: string; + original_vesting: Coin[]; + start_time: number; + end_time: number; + }[] + ) { + this.accountBalance = account_balance; + this.delegationBalance = delegation_balance; + this.unbondingBalance = unbonding_balance; + this.rewardBalance = reward_balance; + this.vesting_account = vesting_account; + } } - From 86fb48fe4edd6032ef2b03edab61aa64bd372bff Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 14 Oct 2022 18:24:46 +0100 Subject: [PATCH 056/224] Fix endpoint --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45704960..1c2fcb0f 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ The only way to derive this figure from the Cosmos SDK APIs is by iterating over #### Endpoint -[`data-api.cheqd.io/staking/balances/`](https://data-api.cheqd.io/staking/balances/cheqdvaloper1lg0vwuu888hu4arnt9egtqrm2662kcrtf2unrs) +[`data-api.cheqd.io/staking/delegators/`](https://data-api.cheqd.io/staking/delegators/cheqdvaloper1lg0vwuu888hu4arnt9egtqrm2662kcrtf2unrs) #### Response From 5047784cf0a95618fb368e4dc74c016b80812215 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:38:27 +0300 Subject: [PATCH 057/224] fix total balance --- src/handlers/totalBalance.ts | 46 ++++++++++++------------------------ 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 4be2220d..19a1c1f7 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -4,9 +4,6 @@ import { validate_cheqd_address, } from '../helpers/validate'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; -import { BigDipperApi } from '../api/bigDipperApi'; -import { GraphQLClient } from '../helpers/graphql'; -import { total_balance_ncheq } from '../helpers/node'; import { NodeApi } from '../api/nodeApi'; export async function handler(request: Request): Promise { @@ -19,32 +16,19 @@ export async function handler(request: Request): Promise { let node_api = new NodeApi(REST_API); let auth_account = await node_api.auth_get_account(address); - if (is_delayed_vesting_account_type(auth_account?.['@type'])) { - let balance = Number( - (await ( - await node_api.bank_get_account_balances(address) - ).find((b) => b.denom === 'ncheq')?.amount) ?? '0' - ); - let rewards = Number( - (await await node_api.distribution_get_total_rewards(address)) ?? '0' - ); - let delegated = Number( - auth_account?.base_vesting_account?.delegated_vesting?.find( - (d) => d.denom === 'ncheq' - )?.amount ?? '0' - ); - - return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); - } - - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); - - let account = await bd_api.get_account(address); - - if (!account) { - throw new Error('Account not found'); - } - - return new Response(ncheq_to_cheq_fixed(total_balance_ncheq(account))); + let balance = Number( + (await ( + await node_api.bank_get_account_balances(address) + ).find((b) => b.denom === 'ncheq')?.amount) ?? '0' + ); + let rewards = Number( + (await await node_api.distribution_get_total_rewards(address)) ?? '0' + ); + let delegated = Number( + auth_account?.base_vesting_account?.delegated_vesting?.find( + (d) => d.denom === 'ncheq' + )?.amount ?? '0' + ); + + return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); } From d7dfca05ac87a5526982d274e775fe139ac4da49 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 18 Oct 2022 15:03:51 +0300 Subject: [PATCH 058/224] fix circulating supply --- src/bindings.d.ts | 1 + src/handlers/balanceGroup.ts | 16 ----- src/handlers/circulatingSupply.ts | 114 ++++++++++++++++++------------ src/handlers/totalBalance.ts | 5 +- src/handlers/webhookTriggers.ts | 2 + src/helpers/balance.ts | 80 +++++++++++---------- src/helpers/balanceGroup.ts | 80 +++++++++++++-------- wrangler.toml | 10 +-- 8 files changed, 170 insertions(+), 138 deletions(-) delete mode 100644 src/handlers/balanceGroup.ts diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 32c54676..d0e56a66 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -3,6 +3,7 @@ declare global { const REST_API: string; const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; + const CURRENT_CSW_GROUP_TO_BE_UPDATED: KVNamespace; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; } diff --git a/src/handlers/balanceGroup.ts b/src/handlers/balanceGroup.ts deleted file mode 100644 index 65109855..00000000 --- a/src/handlers/balanceGroup.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Request } from "itty-router"; -import { updateGroupBalances } from "../helpers/balanceGroup"; - -export async function handler(request: Request): Promise { - const grp = request.params?.['grp']; - - if (grp !== null) { - console.log(`updating all account balances (group: ${grp})`) - const result = await updateGroupBalances(Number(grp), {} as Event) - - console.log({ result }) - return new Response(JSON.stringify(result)); - } - - return new Response("group is missing") -} diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 8830e8cd..183c9830 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,57 +1,79 @@ -import { GraphQLClient } from "../helpers/graphql"; -import { BigDipperApi } from "../api/bigDipperApi"; -import { Request } from "itty-router"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; -import { total_balance_ncheq } from "../helpers/node"; -import { Account } from "../types/bigDipper"; +import { GraphQLClient } from '../helpers/graphql'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { Request } from 'itty-router'; +import { ncheq_to_cheqd, ncheq_to_cheq_fixed } from '../helpers/currency'; +import { total_balance_ncheq } from '../helpers/node'; +import { Account } from '../types/bigDipper'; +import { updateCachedBalance } from '../helpers/balance'; +import { NodeApi } from '../api/nodeApi'; async function get_circulating_supply(): Promise { - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); - - let total_supply = await bd_api.get_total_supply(); - let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); - - try { - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list() - console.log(`found ${cached.keys.length} cached items`) - - let non_circulating_supply_ncheq = Number(0); - for (const r of cached.keys) { - console.log(`looking for account: ${r.name} in cache`) - let data: any = await CIRCULATING_SUPPLY_WATCHLIST.get(r.name, { type: "json" }); - - if (data !== null) { - if (data.totalBalance === undefined) { - const balance = total_balance_ncheq(JSON.parse(data) as Account) - data = JSON.stringify({ totalBalance: balance, updatedAt: Date.now() }) - console.log(`updating bad cache entry: ${JSON.stringify(data)} totalBalance=${data.totalBalance} data=${JSON.stringify(data)}`) - await CIRCULATING_SUPPLY_WATCHLIST.put(r.name, data) - } - - console.log(`found cache entry: ${JSON.stringify(data)} totalBalance=${data.totalBalance}`) - - if (data.totalBalance !== null) { - non_circulating_supply_ncheq += data.totalBalance; - } - } + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); + let node_api = new NodeApi(REST_API); + // TODO refactor all of this ex: gettin total supply , gettin total staked + + let total_supply_ncheq = await node_api.bank_get_total_supply_ncheq(); + let total_staked_coins_ncheq = await bd_api.get_total_staked_coins(); + const total_staked_coins = Number( + ncheq_to_cheq_fixed(Number(total_staked_coins_ncheq)) + ); + const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); + + try { + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); + console.log(`found ${cached.keys.length} cached items`); + + let non_circulating_supply = Number(0); + for (const r of cached.keys) { + console.log(`looking for account: ${r.name} in cache`); + let data: any = await CIRCULATING_SUPPLY_WATCHLIST.get(r.name, { + type: 'json', + }); + + if (data !== null) { + if (data.totalBalance === undefined) { + const balance = + // TODO change this to commented implemntation. + // updateCachedBalance(node_api, r.name.split("grp:")[1], ); + total_balance_ncheq(JSON.parse(data) as Account); + data = JSON.stringify({ + totalBalance: balance, + updatedAt: Date.now(), + }); + console.log( + `updating bad cache entry: ${JSON.stringify(data)} totalBalance=${ + data.totalBalance + } data=${JSON.stringify(data)}` + ); + await CIRCULATING_SUPPLY_WATCHLIST.put(r.name, data); } - console.log(`Non-circulating supply: ${non_circulating_supply_ncheq}`); - // Get total supply - let total_supply_ncheq = Number(total_supply.find(c => c.denom === "ncheq")?.amount || '0'); - console.log(`Total supply: ${total_supply_ncheq}`); + console.log( + `found cache entry: ${JSON.stringify(data)} totalBalance=${ + data.totalBalance + }` + ); - // Calculate circulating supply - return total_supply_ncheq - non_circulating_supply_ncheq; - } catch (e: any) { - console.error(new Map(e)) - return total_supply_ncheq + if (data.totalBalance !== null) { + non_circulating_supply += Number(data.totalBalance); + console.log('non circulatin ', non_circulating_supply); + } + } } + + console.log(`Non-circulating supply: ${non_circulating_supply}`); + console.log('Total staked coins', total_staked_coins); + console.log('Total supply', total_supply); + return total_supply - (total_staked_coins + non_circulating_supply); + } catch (e: any) { + console.error(new Map(e)); + return total_supply_ncheq; + } } export async function handler(request: Request): Promise { - let circulating_supply = await get_circulating_supply(); + let circulating_supply = await get_circulating_supply(); - return new Response(ncheq_to_cheq_fixed(circulating_supply)); + return new Response(ncheq_to_cheq_fixed(circulating_supply)); } diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 19a1c1f7..184a4e96 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,8 +1,5 @@ import { Request } from 'itty-router'; -import { - is_delayed_vesting_account_type, - validate_cheqd_address, -} from '../helpers/validate'; +import { validate_cheqd_address } from '../helpers/validate'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; import { NodeApi } from '../api/nodeApi'; diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index f674601e..051791ab 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,8 +1,10 @@ +import { updateGroupBalances } from '../helpers/balanceGroup'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); + await updateGroupBalances(Number(3)); } export async function sendPriceDiscrepancies() { diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 454459e9..4a1ca527 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,37 +1,45 @@ -import { NodeApi } from "../api/nodeApi"; -import { GraphQLClient } from "./graphql"; -import { BigDipperApi } from "../api/bigDipperApi"; -import { total_balance_ncheq } from "./node"; -import { Account } from "../types/bigDipper"; - -export async function updateCachedBalance(node_api: NodeApi, addr: string, grpN: number): Promise { - const gql_client = new GraphQLClient(GRAPHQL_API); - const bd_api = new BigDipperApi(gql_client); - const account = await bd_api.get_account(addr); - - if (!account) { - throw new Error(`Account not found for address "${addr}"`) - } - - try { - const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`, { type: "json" }) - - // if (cachedAccount !== undefined) { - console.log(`account "${addr}" found in cache: ${JSON.stringify(cachedAccount)}`) - - const totalBalance = total_balance_ncheq(account); - const data = JSON.stringify({ totalBalance: totalBalance }); - - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data) - - console.log(`account "${addr}" balance updated. (${data})`) - - return account; - // } - - return account; - } catch (e: any) { - console.error(new Map(e)) - return null; - } +import { NodeApi } from '../api/nodeApi'; +import { GraphQLClient } from './graphql'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { total_balance_ncheq } from './node'; +import { Account } from '../types/bigDipper'; +import { ncheq_to_cheq_fixed } from './currency'; + +export async function updateCachedBalance( + node_api: NodeApi, + addr: string, + grpN: number +) { + let balance = Number( + (await ( + await node_api.bank_get_account_balances(addr) + ).find((b) => b.denom === 'ncheq')?.amount) ?? '0' + ); + let rewards = Number( + (await await node_api.distribution_get_total_rewards(addr)) ?? '0' + ); + + try { + const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get( + `grp_${grpN}:${addr}`, + { type: 'json' } + ); + + // if (cachedAccount !== undefined) { + console.log( + `account "${addr}" found in cache: ${JSON.stringify(cachedAccount)}` + ); + + const totalBalance = ncheq_to_cheq_fixed(balance + rewards); + const data = JSON.stringify({ totalBalance: totalBalance }); + + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data); + + console.log(`account "${addr}" balance updated. (${data})`); + + // } + } catch (e: any) { + console.error(new Map(e)); + // return null; + } } diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index ce346564..38197c39 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -1,36 +1,54 @@ -import { updateCachedBalance } from "./balance"; -import { NodeApi } from "../api/nodeApi"; -import { Account } from "../types/bigDipper"; - -export async function updateGroupBalances(group: number, event: Event) { - let node_api = new NodeApi(REST_API); - let balances: { account: Account } [] = []; - - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${group}:` }); - - console.log(`found ${cached.keys.length} cached accounts`) - - for (const key of cached.keys) { - const parts = key.name.split(':') - let addr = parts[1] - let grpN = Number(parts[0].split("_")[1]) - - if (key.name.includes("delayed:")) { - addr = parts[2] - } +import { updateCachedBalance } from './balance'; +import { NodeApi } from '../api/nodeApi'; +import { Account } from '../types/bigDipper'; + +export async function updateGroupBalances() { + console.log('updatingg....'); + let node_api = new NodeApi(REST_API); + let grouptToBeUpdated = await CURRENT_CSW_GROUP_TO_BE_UPDATED.get('group'); + + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ + prefix: `grp_${Number(grouptToBeUpdated)}:`, + }); + + console.log( + `found ${cached.keys.length} cached accounts for group ${grouptToBeUpdated}` + ); + + for (const key of cached.keys) { + const parts = key.name.split(':'); + let addr = parts[1]; + let grpN = Number(parts[0].split('_')[1]); + + if (key.name.includes('delayed:')) { + addr = parts[2]; + console.log('some include delayed...'); + } - const found = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`) - if (found) { - console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`) + const found = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`); + console.log('address to be updated', found); + if (found) { + console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`); - const account = await updateCachedBalance(node_api, addr, grpN) + const account = await updateCachedBalance(node_api, addr, grpN); - if (account !== null) { - console.log(`updating account (grp_${grpN}:${addr}) balance (${JSON.stringify(account)})`) - balances.push({ account: account }) - } - } + if (account !== null) { + console.log( + `updating account (grp_${grpN}:${addr}) balance (${JSON.stringify( + account + )})` + ); + } } - - return balances + } + // TODO: move group totatl to env var + // updated CSW_group + if (Number(grouptToBeUpdated) < 4) { + await CURRENT_CSW_GROUP_TO_BE_UPDATED.put( + 'group', + `${Number(grouptToBeUpdated) + 1}` + ); + } else { + await CURRENT_CSW_GROUP_TO_BE_UPDATED.put('group', `${1}`); + } } diff --git a/wrangler.toml b/wrangler.toml index 000e6811..4420db8d 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -96,14 +96,14 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # KV Namespaces accessible from the Worker # Details: https://developers.cloudflare.com/workers/learning/how-kv-works # @default `[]` -[[env.staging.kv_namespaces]] -binding = "CIRCULATING_SUPPLY_WATCHLIST" -id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2" -preview_id = "1e1032cbf6854d88b12317da8a792928" + +kv_namespaces = [ + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2"} , { binding = "CURRENT_CSW_GROUP_TO_BE_UPDATED", id = "da8c7da9529943c79167ca1254abc3cc" } +] # Cron triggers for staging worker [env.staging.triggers] -crons = ["0 9 * * *"] +crons = ["* * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From d07d0ee3b0b9cf608fb227eb8dc167113717443c Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 18 Oct 2022 16:17:52 +0300 Subject: [PATCH 059/224] chore refactor a bit --- src/handlers/circulatingSupply.ts | 58 +++++++++++++++---------------- src/handlers/webhookTriggers.ts | 2 +- src/helpers/balance.ts | 40 +++++++++------------ src/helpers/balanceGroup.ts | 43 +++++++++++++---------- 4 files changed, 70 insertions(+), 73 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 183c9830..d39508e9 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,18 +1,18 @@ import { GraphQLClient } from '../helpers/graphql'; import { BigDipperApi } from '../api/bigDipperApi'; import { Request } from 'itty-router'; -import { ncheq_to_cheqd, ncheq_to_cheq_fixed } from '../helpers/currency'; -import { total_balance_ncheq } from '../helpers/node'; -import { Account } from '../types/bigDipper'; +import { ncheq_to_cheq_fixed } from '../helpers/currency'; import { updateCachedBalance } from '../helpers/balance'; import { NodeApi } from '../api/nodeApi'; +import { extract_group_number_and_address } from '../helpers/balanceGroup'; -async function get_circulating_supply(): Promise { +async function get_total_supply_and_total_staked_coins_in_cheqd(): Promise<{ + total_staked_coins: number; + total_supply: number; +}> { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); let node_api = new NodeApi(REST_API); - // TODO refactor all of this ex: gettin total supply , gettin total staked - let total_supply_ncheq = await node_api.bank_get_total_supply_ncheq(); let total_staked_coins_ncheq = await bd_api.get_total_staked_coins(); const total_staked_coins = Number( @@ -20,33 +20,32 @@ async function get_circulating_supply(): Promise { ); const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); + return { + total_staked_coins, + total_supply, + }; +} + +async function get_circulating_supply(): Promise { + let node_api = new NodeApi(REST_API); + const { total_staked_coins, total_supply } = + await get_total_supply_and_total_staked_coins_in_cheqd(); + try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); console.log(`found ${cached.keys.length} cached items`); - let non_circulating_supply = Number(0); - for (const r of cached.keys) { - console.log(`looking for account: ${r.name} in cache`); - let data: any = await CIRCULATING_SUPPLY_WATCHLIST.get(r.name, { + let shareholders_total_balance = Number(0); + for (const key of cached.keys) { + console.log(`looking for account: ${key.name} in cache`); + let data: any = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name, { type: 'json', }); if (data !== null) { if (data.totalBalance === undefined) { - const balance = - // TODO change this to commented implemntation. - // updateCachedBalance(node_api, r.name.split("grp:")[1], ); - total_balance_ncheq(JSON.parse(data) as Account); - data = JSON.stringify({ - totalBalance: balance, - updatedAt: Date.now(), - }); - console.log( - `updating bad cache entry: ${JSON.stringify(data)} totalBalance=${ - data.totalBalance - } data=${JSON.stringify(data)}` - ); - await CIRCULATING_SUPPLY_WATCHLIST.put(r.name, data); + const parts = extract_group_number_and_address(key.name); + updateCachedBalance(node_api, parts.address, parts.groupNumber); } console.log( @@ -56,24 +55,23 @@ async function get_circulating_supply(): Promise { ); if (data.totalBalance !== null) { - non_circulating_supply += Number(data.totalBalance); - console.log('non circulatin ', non_circulating_supply); + shareholders_total_balance += Number(data.totalBalance); + console.log('current non circulating', shareholders_total_balance); } } } - console.log(`Non-circulating supply: ${non_circulating_supply}`); + console.log(`Shareholders total balance: ${shareholders_total_balance}`); console.log('Total staked coins', total_staked_coins); console.log('Total supply', total_supply); - return total_supply - (total_staked_coins + non_circulating_supply); + return total_supply - (total_staked_coins + shareholders_total_balance); } catch (e: any) { console.error(new Map(e)); - return total_supply_ncheq; } } export async function handler(request: Request): Promise { let circulating_supply = await get_circulating_supply(); - return new Response(ncheq_to_cheq_fixed(circulating_supply)); + return new Response(circulating_supply.toString()); } diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 051791ab..549dea97 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -4,7 +4,7 @@ import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); - await updateGroupBalances(Number(3)); + await updateGroupBalances(); } export async function sendPriceDiscrepancies() { diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 4a1ca527..78c1f965 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,15 +1,10 @@ import { NodeApi } from '../api/nodeApi'; -import { GraphQLClient } from './graphql'; -import { BigDipperApi } from '../api/bigDipperApi'; -import { total_balance_ncheq } from './node'; -import { Account } from '../types/bigDipper'; import { ncheq_to_cheq_fixed } from './currency'; -export async function updateCachedBalance( +async function calculate_total_balance_without_delegations( node_api: NodeApi, - addr: string, - grpN: number -) { + addr: string +): Promise { let balance = Number( (await ( await node_api.bank_get_account_balances(addr) @@ -18,28 +13,25 @@ export async function updateCachedBalance( let rewards = Number( (await await node_api.distribution_get_total_rewards(addr)) ?? '0' ); - + return Number(ncheq_to_cheq_fixed(balance + rewards)); +} +export async function updateCachedBalance( + node_api: NodeApi, + addr: string, + grpN: number +) { try { - const cachedAccount = await CIRCULATING_SUPPLY_WATCHLIST.get( - `grp_${grpN}:${addr}`, - { type: 'json' } - ); - - // if (cachedAccount !== undefined) { - console.log( - `account "${addr}" found in cache: ${JSON.stringify(cachedAccount)}` - ); + const total_balance_without_delegations = + await calculate_total_balance_without_delegations(node_api, addr); - const totalBalance = ncheq_to_cheq_fixed(balance + rewards); - const data = JSON.stringify({ totalBalance: totalBalance }); + const data = JSON.stringify({ + totalBalance: total_balance_without_delegations, + }); await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data); console.log(`account "${addr}" balance updated. (${data})`); - - // } } catch (e: any) { - console.error(new Map(e)); - // return null; + console.error('error', e); } } diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index 38197c39..7caf314a 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -1,32 +1,39 @@ import { updateCachedBalance } from './balance'; import { NodeApi } from '../api/nodeApi'; -import { Account } from '../types/bigDipper'; + +export function extract_group_number_and_address(key: string) { + const parts = key.split(':'); + let addr = parts[1]; + let grpN = Number(parts[0].split('_')[1]); + + if (key.includes('delayed:')) { + addr = parts[2]; + } + return { + address: addr, + groupNumber: grpN, + }; +} export async function updateGroupBalances() { - console.log('updatingg....'); let node_api = new NodeApi(REST_API); - let grouptToBeUpdated = await CURRENT_CSW_GROUP_TO_BE_UPDATED.get('group'); - + let balance_group_to_be_updated = await CURRENT_CSW_GROUP_TO_BE_UPDATED.get( + 'group' + ); const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: `grp_${Number(grouptToBeUpdated)}:`, + prefix: `grp_${balance_group_to_be_updated}:`, }); console.log( - `found ${cached.keys.length} cached accounts for group ${grouptToBeUpdated}` + `found ${cached.keys.length} cached accounts for group ${balance_group_to_be_updated}` ); for (const key of cached.keys) { - const parts = key.name.split(':'); - let addr = parts[1]; - let grpN = Number(parts[0].split('_')[1]); - - if (key.name.includes('delayed:')) { - addr = parts[2]; - console.log('some include delayed...'); - } + const parts = extract_group_number_and_address(key.name); + let addr = parts.address; + let grpN = parts.groupNumber; const found = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`); - console.log('address to be updated', found); if (found) { console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`); @@ -42,11 +49,11 @@ export async function updateGroupBalances() { } } // TODO: move group totatl to env var - // updated CSW_group - if (Number(grouptToBeUpdated) < 4) { + // updated CSW_group (note: we have 4 groups as of now) + if (Number(balance_group_to_be_updated) < 4) { await CURRENT_CSW_GROUP_TO_BE_UPDATED.put( 'group', - `${Number(grouptToBeUpdated) + 1}` + `${Number(balance_group_to_be_updated) + 1}` ); } else { await CURRENT_CSW_GROUP_TO_BE_UPDATED.put('group', `${1}`); From 74a67fb3c48a0b9feb4938e9bf670b74aa24d902 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:22:59 +0300 Subject: [PATCH 060/224] fix node api (include unbounding/delegated free balance for totalBalance --- src/handlers/totalBalance.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 184a4e96..784785ba 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -24,8 +24,15 @@ export async function handler(request: Request): Promise { let delegated = Number( auth_account?.base_vesting_account?.delegated_vesting?.find( (d) => d.denom === 'ncheq' - )?.amount ?? '0' + )?.amount ?? '0'; + + let unbonding = Number( + auth_account?.base_vesting_account?.delegated_free?.find( + (d) => d.denom === 'ncheq' + )?.amount ?? '0' + ); + - return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); + return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding)); } From 452842532421d9e279d8aaf9676406d00103806e Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:32:06 +0300 Subject: [PATCH 061/224] make total balance endpoint to consider unbounding balance --- src/handlers/circulatingSupply.ts | 2 +- src/handlers/totalBalance.ts | 36 ++++++------------------------- src/helpers/balance.ts | 29 +++++++++++++++++++------ 3 files changed, 30 insertions(+), 37 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index d39508e9..cde3cdd9 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -64,7 +64,7 @@ async function get_circulating_supply(): Promise { console.log(`Shareholders total balance: ${shareholders_total_balance}`); console.log('Total staked coins', total_staked_coins); console.log('Total supply', total_supply); - return total_supply - (total_staked_coins + shareholders_total_balance); + return total_supply - shareholders_total_balance; } catch (e: any) { console.error(new Map(e)); } diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 784785ba..0457eb4e 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,38 +1,14 @@ import { Request } from 'itty-router'; -import { validate_cheqd_address } from '../helpers/validate'; -import { ncheq_to_cheq_fixed } from '../helpers/currency'; import { NodeApi } from '../api/nodeApi'; +import { calculate_total_balance_for_an_account } from '../helpers/balance'; export async function handler(request: Request): Promise { const address = request.params?.['address']; - - if (!address || !validate_cheqd_address(address)) { - throw new Error('No address specified or wrong address format.'); - } - - let node_api = new NodeApi(REST_API); - let auth_account = await node_api.auth_get_account(address); - - let balance = Number( - (await ( - await node_api.bank_get_account_balances(address) - ).find((b) => b.denom === 'ncheq')?.amount) ?? '0' - ); - let rewards = Number( - (await await node_api.distribution_get_total_rewards(address)) ?? '0' - ); - let delegated = Number( - auth_account?.base_vesting_account?.delegated_vesting?.find( - (d) => d.denom === 'ncheq' - )?.amount ?? '0'; - - let unbonding = Number( - auth_account?.base_vesting_account?.delegated_free?.find( - (d) => d.denom === 'ncheq' - )?.amount ?? '0' - + const node_api = new NodeApi(REST_API); + const total_balance = await calculate_total_balance_for_an_account( + node_api, + address!! ); - - return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding)); + return new Response(total_balance.toString()); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 78c1f965..8076ea9c 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,19 +1,36 @@ import { NodeApi } from '../api/nodeApi'; import { ncheq_to_cheq_fixed } from './currency'; +import { validate_cheqd_address } from './validate'; -async function calculate_total_balance_without_delegations( +export async function calculate_total_balance_for_an_account( node_api: NodeApi, - addr: string + address: string ): Promise { + if (!address || !validate_cheqd_address(address)) { + throw new Error('No address specified or wrong address format.'); + } + let auth_account = await node_api.auth_get_account(address); + let balance = Number( (await ( - await node_api.bank_get_account_balances(addr) + await node_api.bank_get_account_balances(address) ).find((b) => b.denom === 'ncheq')?.amount) ?? '0' ); let rewards = Number( - (await await node_api.distribution_get_total_rewards(addr)) ?? '0' + (await await node_api.distribution_get_total_rewards(address)) ?? '0' + ); + let delegated = Number( + auth_account?.base_vesting_account?.delegated_vesting?.find( + (d) => d.denom === 'ncheq' + )?.amount ?? '0' + ); + + let unbonding = Number( + auth_account?.base_vesting_account?.delegated_free?.find( + (d) => d.denom === 'ncheq' + )?.amount ?? '0' ); - return Number(ncheq_to_cheq_fixed(balance + rewards)); + return Number(ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding)); } export async function updateCachedBalance( node_api: NodeApi, @@ -22,7 +39,7 @@ export async function updateCachedBalance( ) { try { const total_balance_without_delegations = - await calculate_total_balance_without_delegations(node_api, addr); + await calculate_total_balance_for_an_account(node_api, addr); const data = JSON.stringify({ totalBalance: total_balance_without_delegations, From 921a8869be6ab59a7f70a76c198b59dca87bd2c2 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Tue, 18 Oct 2022 17:06:22 +0100 Subject: [PATCH 062/224] Squashed commit of the following: commit ff235a6637ea72311ab45d72d81e5c825e9dd38f Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue Oct 18 11:08:46 2022 +0100 chore(deps-dev): Bump @cloudflare/workers-types from 3.16.0 to 3.17.0 (#36) Bumps [@cloudflare/workers-types](https://github.com/cloudflare/workers-types) from 3.16.0 to 3.17.0. - [Release notes](https://github.com/cloudflare/workers-types/releases) - [Changelog](https://github.com/cloudflare/workers-types/blob/master/CHANGELOG.md) - [Commits](https://github.com/cloudflare/workers-types/compare/v3.16.0...v3.17.0) --- updated-dependencies: - dependency-name: "@cloudflare/workers-types" dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index fc94a070..8b133935 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "itty-router": "^2.6.6" }, "devDependencies": { - "@cloudflare/workers-types": "^3.16.0", + "@cloudflare/workers-types": "^3.17.0", "@types/node": "^17.0.45", "typescript": "^4.8.4", "wrangler": "^2.1.11" @@ -28,9 +28,9 @@ } }, "node_modules/@cloudflare/workers-types": { - "version": "3.16.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-3.16.0.tgz", - "integrity": "sha512-gaBUSaKS65mN3iKZEgichbXYEmAa/pXkc5Gbt+1BptYphdGkj09ggdsiE4w8g0F/uI1g36QaTKrzVnBAWMipvQ==", + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-3.17.0.tgz", + "integrity": "sha512-u0cUQ4ntWFFwn5jx0ETa2ItvwvfOMjyaKF2fX2vFVujrSgNES/PnvRzPAhdt9CMYAMidInm0MGkIjxHRsFBaeg==", "dev": true }, "node_modules/@esbuild-plugins/node-globals-polyfill": { @@ -1570,9 +1570,9 @@ } }, "@cloudflare/workers-types": { - "version": "3.16.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-3.16.0.tgz", - "integrity": "sha512-gaBUSaKS65mN3iKZEgichbXYEmAa/pXkc5Gbt+1BptYphdGkj09ggdsiE4w8g0F/uI1g36QaTKrzVnBAWMipvQ==", + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-3.17.0.tgz", + "integrity": "sha512-u0cUQ4ntWFFwn5jx0ETa2ItvwvfOMjyaKF2fX2vFVujrSgNES/PnvRzPAhdt9CMYAMidInm0MGkIjxHRsFBaeg==", "dev": true }, "@esbuild-plugins/node-globals-polyfill": { diff --git a/package.json b/package.json index 6199e419..9c0ae8f5 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "itty-router": "^2.6.6" }, "devDependencies": { - "@cloudflare/workers-types": "^3.16.0", + "@cloudflare/workers-types": "^3.17.0", "@types/node": "^17.0.45", "typescript": "^4.8.4", "wrangler": "^2.1.11" From 5f955e14d1dcbea510279987d1a29b5c1ab4a65b Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:34:45 +0300 Subject: [PATCH 063/224] chore store account balance infos in KV --- src/handlers/circulatingSupply.ts | 46 ++++++++++++++----------------- src/handlers/totalBalance.ts | 6 ++-- src/helpers/balance.ts | 26 +++++++++++------ src/types/node.ts | 9 ++++++ 4 files changed, 50 insertions(+), 37 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index cde3cdd9..e9df8506 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,35 +1,21 @@ -import { GraphQLClient } from '../helpers/graphql'; -import { BigDipperApi } from '../api/bigDipperApi'; import { Request } from 'itty-router'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; import { updateCachedBalance } from '../helpers/balance'; import { NodeApi } from '../api/nodeApi'; import { extract_group_number_and_address } from '../helpers/balanceGroup'; +import { AccountBalanceInfos } from '../types/node'; -async function get_total_supply_and_total_staked_coins_in_cheqd(): Promise<{ - total_staked_coins: number; - total_supply: number; -}> { - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); +async function get_total_supply(): Promise { let node_api = new NodeApi(REST_API); let total_supply_ncheq = await node_api.bank_get_total_supply_ncheq(); - let total_staked_coins_ncheq = await bd_api.get_total_staked_coins(); - const total_staked_coins = Number( - ncheq_to_cheq_fixed(Number(total_staked_coins_ncheq)) - ); const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); - return { - total_staked_coins, - total_supply, - }; + return total_supply; } async function get_circulating_supply(): Promise { let node_api = new NodeApi(REST_API); - const { total_staked_coins, total_supply } = - await get_total_supply_and_total_staked_coins_in_cheqd(); + const total_supply = await get_total_supply(); try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); @@ -38,12 +24,20 @@ async function get_circulating_supply(): Promise { let shareholders_total_balance = Number(0); for (const key of cached.keys) { console.log(`looking for account: ${key.name} in cache`); - let data: any = await CIRCULATING_SUPPLY_WATCHLIST.get(key.name, { - type: 'json', - }); + let data: AccountBalanceInfos | null = + await CIRCULATING_SUPPLY_WATCHLIST.get(key.name, { + type: 'json', + }); if (data !== null) { - if (data.totalBalance === undefined) { + if ( + data.totalBalance === undefined || + data.availiableBalance === undefined || + data.rewards === undefined || + data.delegated === undefined || + data.unbounding === undefined || + data.timeUpdated === undefined + ) { const parts = extract_group_number_and_address(key.name); updateCachedBalance(node_api, parts.address, parts.groupNumber); } @@ -56,17 +50,17 @@ async function get_circulating_supply(): Promise { if (data.totalBalance !== null) { shareholders_total_balance += Number(data.totalBalance); - console.log('current non circulating', shareholders_total_balance); } } } - console.log(`Shareholders total balance: ${shareholders_total_balance}`); - console.log('Total staked coins', total_staked_coins); + console.log( + `Sum of Shareholders total balance: ${shareholders_total_balance}` + ); console.log('Total supply', total_supply); return total_supply - shareholders_total_balance; } catch (e: any) { - console.error(new Map(e)); + throw new Error(e.toString); } } diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 0457eb4e..b4f6e48f 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,14 +1,14 @@ import { Request } from 'itty-router'; import { NodeApi } from '../api/nodeApi'; -import { calculate_total_balance_for_an_account } from '../helpers/balance'; +import { get_account_balance_infos } from '../helpers/balance'; export async function handler(request: Request): Promise { const address = request.params?.['address']; const node_api = new NodeApi(REST_API); - const total_balance = await calculate_total_balance_for_an_account( + const account_balance_infos = await get_account_balance_infos( node_api, address!! ); - return new Response(total_balance.toString()); + return new Response(account_balance_infos.totalBalance.toString()); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 8076ea9c..38a90aba 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,11 +1,12 @@ import { NodeApi } from '../api/nodeApi'; +import { AccountBalanceInfos } from '../types/node'; import { ncheq_to_cheq_fixed } from './currency'; import { validate_cheqd_address } from './validate'; -export async function calculate_total_balance_for_an_account( +export async function get_account_balance_infos( node_api: NodeApi, address: string -): Promise { +): Promise { if (!address || !validate_cheqd_address(address)) { throw new Error('No address specified or wrong address format.'); } @@ -30,7 +31,16 @@ export async function calculate_total_balance_for_an_account( (d) => d.denom === 'ncheq' )?.amount ?? '0' ); - return Number(ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding)); + return { + totalBalance: Number( + ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding) + ), + availiableBalance: balance, + rewards: Number(ncheq_to_cheq_fixed(rewards)), + delegated: Number(ncheq_to_cheq_fixed(delegated)), + unbounding: Number(ncheq_to_cheq_fixed(unbonding)), + timeUpdated: new Date().toUTCString(), + }; } export async function updateCachedBalance( node_api: NodeApi, @@ -38,12 +48,12 @@ export async function updateCachedBalance( grpN: number ) { try { - const total_balance_without_delegations = - await calculate_total_balance_for_an_account(node_api, addr); + const account_balance_infos = await get_account_balance_infos( + node_api, + addr + ); - const data = JSON.stringify({ - totalBalance: total_balance_without_delegations, - }); + const data = JSON.stringify(account_balance_infos); await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data); diff --git a/src/types/node.ts b/src/types/node.ts index 16faa481..4edd481b 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -76,3 +76,12 @@ export interface TotalStakedCoinsResponse { } ]; } + +export interface AccountBalanceInfos { + totalBalance: number; + availiableBalance: number; + rewards: number; + delegated: number; + unbounding: number; + timeUpdated: string; +} From 2e34644626a354d7f7322eea8f3fc76ea98c863b Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:20:21 +0300 Subject: [PATCH 064/224] fix total balance --- src/api/bigDipperApi.ts | 53 +++++++----------- src/api/nodeApi.ts | 8 +++ src/handlers/totalBalance.ts | 8 +-- src/helpers/balance.ts | 103 ++++++++++++++++++++++------------- 4 files changed, 96 insertions(+), 76 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 17bbf013..57f2c7df 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -5,45 +5,34 @@ import { TotalStakedCoinsResponse, ValidatorDelegationsCountResponse, } from '../types/node'; -import { Account } from '../types/bigDipper'; +import { Account, LatestBlockHeightResponse } from '../types/bigDipper'; import { NodeApi } from './nodeApi'; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} - async get_account(address: string): Promise { - let query = `query Account($address: String!, $where: vesting_account_bool_exp) { - accountBalance: action_account_balance(address: $address) { - coins - } - delegationBalance: action_delegation_total(address: $address) { - coins - } - unbondingBalance: action_unbonding_delegation_total(address: $address) { - coins - } - redelegationBalance: action_redelegation(address: $address) { - redelegations - } - rewardBalance: action_delegation_reward(address: $address) { - coins - } - vesting_account(where: $where) { - id - type - original_vesting - start_time - end_time - } - }`; + async get_account(address: string, height: number): Promise { + let query = `query Account($address: String!, $height: Int!) { + accountBalance: action_account_balance(address: $address, height: $height) { + coins + } + delegationBalance: action_delegation_total(address: $address, height: $height) { + coins + } + unbondingBalance: action_unbonding_delegation_total(address: $address, height: $height) { + coins + } + redelegationBalance: action_redelegation(address: $address, height: $height) { + redelegations + } + rewardBalance: action_delegation_reward(address: $address, height: $height) { + coins + } + }`; let params = { address: address, - where: { - address: { - _eq: address, - }, - }, + height: height, }; try { @@ -54,7 +43,7 @@ export class BigDipperApi { return resp.data as Account; } catch (e: any) { - console.error(new Map(e)); + console.log(e); return null; } } diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 1e53d2f3..eca22c04 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -51,4 +51,12 @@ export class NodeApi { return await resp.json(); } + + async get_latest_block_height(): Promise { + const resp = await fetch(`${this.base_rest_api_url}/blocks/latest`); + let respJson = (await resp.json()) as { + block: { header: { height: number } }; + }; + return Number(respJson.block.header.height); + } } diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index b4f6e48f..536f5ba9 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -4,11 +4,7 @@ import { get_account_balance_infos } from '../helpers/balance'; export async function handler(request: Request): Promise { const address = request.params?.['address']; - const node_api = new NodeApi(REST_API); - const account_balance_infos = await get_account_balance_infos( - node_api, - address!! - ); + const account_balance_infos = await get_account_balance_infos(address!!); - return new Response(account_balance_infos.totalBalance.toString()); + return new Response(account_balance_infos?.totalBalance.toString()); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 38a90aba..31015095 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,57 +1,84 @@ +import { BigDipperApi } from '../api/bigDipperApi'; import { NodeApi } from '../api/nodeApi'; +import { Account } from '../types/bigDipper'; import { AccountBalanceInfos } from '../types/node'; import { ncheq_to_cheq_fixed } from './currency'; -import { validate_cheqd_address } from './validate'; - -export async function get_account_balance_infos( - node_api: NodeApi, - address: string -): Promise { - if (!address || !validate_cheqd_address(address)) { - throw new Error('No address specified or wrong address format.'); - } - let auth_account = await node_api.auth_get_account(address); +import { GraphQLClient } from './graphql'; +function extract_account_infos(account: Account) { let balance = Number( - (await ( - await node_api.bank_get_account_balances(address) - ).find((b) => b.denom === 'ncheq')?.amount) ?? '0' - ); - let rewards = Number( - (await await node_api.distribution_get_total_rewards(address)) ?? '0' - ); - let delegated = Number( - auth_account?.base_vesting_account?.delegated_vesting?.find( - (d) => d.denom === 'ncheq' - )?.amount ?? '0' + account?.accountBalance?.coins.find((c) => c.denom === 'ncheq')?.amount || + '0' ); - let unbonding = Number( - auth_account?.base_vesting_account?.delegated_free?.find( - (d) => d.denom === 'ncheq' - )?.amount ?? '0' - ); + let delegated = 0; + if ( + account?.delegationBalance?.coins && + account?.delegationBalance?.coins.length > 0 + ) { + delegated = Number(account?.delegationBalance?.coins[0].amount); + } + + let unbonding = 0; + if ( + account?.unbondingBalance?.coins && + account?.unbondingBalance?.coins.length > 0 + ) { + unbonding = Number(account?.unbondingBalance?.coins[0]?.amount); + } + + let rewards = 0; + if (account?.rewardBalance?.length > 0) { + for (let i = 0; i < account?.rewardBalance.length; i++) { + rewards += Number(account?.rewardBalance[i]?.coins[0].amount); + } + } + return { - totalBalance: Number( - ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding) - ), - availiableBalance: balance, - rewards: Number(ncheq_to_cheq_fixed(rewards)), - delegated: Number(ncheq_to_cheq_fixed(delegated)), - unbounding: Number(ncheq_to_cheq_fixed(unbonding)), - timeUpdated: new Date().toUTCString(), + balance, + rewards, + delegated, + unbonding, }; } +export async function get_account_balance_infos( + address: string +): Promise { + try { + const gql_client = new GraphQLClient(GRAPHQL_API); + const bd_api = new BigDipperApi(gql_client); + const node_api = new NodeApi(REST_API); + const latest_block_height = await node_api.get_latest_block_height(); + const account_balance_infos: Account | null = await bd_api.get_account( + address, + latest_block_height!! + ); + const { balance, rewards, delegated, unbonding } = extract_account_infos( + account_balance_infos!! + ); + return { + totalBalance: Number( + ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding) + ), + availiableBalance: balance, + rewards: Number(ncheq_to_cheq_fixed(rewards)), + delegated: Number(ncheq_to_cheq_fixed(delegated)), + unbounding: Number(ncheq_to_cheq_fixed(unbonding)), + timeUpdated: new Date().toUTCString(), + }; + } catch (e) { + console.error(e); + return null; + } +} + export async function updateCachedBalance( node_api: NodeApi, addr: string, grpN: number ) { try { - const account_balance_infos = await get_account_balance_infos( - node_api, - addr - ); + const account_balance_infos = await get_account_balance_infos(addr); const data = JSON.stringify(account_balance_infos); From 2573fa359b868cbb29e6f9e90656368769c7d9fb Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:26:07 +0300 Subject: [PATCH 065/224] trigger cron based on current --- src/handlers/circulatingSupply.ts | 3 +-- src/handlers/webhookTriggers.ts | 23 ++++++++++++++++++++++- src/helpers/balance.ts | 6 +----- src/helpers/balanceGroup.ts | 22 +++++----------------- src/index.ts | 21 --------------------- 5 files changed, 29 insertions(+), 46 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index e9df8506..22b4fc10 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -14,7 +14,6 @@ async function get_total_supply(): Promise { } async function get_circulating_supply(): Promise { - let node_api = new NodeApi(REST_API); const total_supply = await get_total_supply(); try { @@ -39,7 +38,7 @@ async function get_circulating_supply(): Promise { data.timeUpdated === undefined ) { const parts = extract_group_number_and_address(key.name); - updateCachedBalance(node_api, parts.address, parts.groupNumber); + updateCachedBalance(parts.address, parts.groupNumber); } console.log( diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 549dea97..4e72a643 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -4,7 +4,7 @@ import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); - await updateGroupBalances(); + await updateGroupBalances(getRandomGroup()); } export async function sendPriceDiscrepancies() { @@ -31,3 +31,24 @@ export async function sendPriceDiscrepancies() { } } } + +function getRandomGroup(): number { + const hour = new Date().getHours(); + + if (hour > 0 && hour < 6) { + return 1; + } + + if (hour >= 6 && hour < 12) { + return 2; + } + + if (hour >= 12 && hour < 18) { + return 3; + } + + if (hour >= 18 && hour < 24) { + return 4; + } + throw new Error('invalid hour for group'); +} diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 31015095..33910354 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -72,11 +72,7 @@ export async function get_account_balance_infos( } } -export async function updateCachedBalance( - node_api: NodeApi, - addr: string, - grpN: number -) { +export async function updateCachedBalance(addr: string, grpN: number) { try { const account_balance_infos = await get_account_balance_infos(addr); diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index 7caf314a..cfedb54d 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -15,17 +15,15 @@ export function extract_group_number_and_address(key: string) { }; } -export async function updateGroupBalances() { +export async function updateGroupBalances(groupNumber: number) { let node_api = new NodeApi(REST_API); - let balance_group_to_be_updated = await CURRENT_CSW_GROUP_TO_BE_UPDATED.get( - 'group' - ); + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: `grp_${balance_group_to_be_updated}:`, + prefix: `grp_${groupNumber}:`, }); console.log( - `found ${cached.keys.length} cached accounts for group ${balance_group_to_be_updated}` + `found ${cached.keys.length} cached accounts for group ${groupNumber}` ); for (const key of cached.keys) { @@ -37,7 +35,7 @@ export async function updateGroupBalances() { if (found) { console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`); - const account = await updateCachedBalance(node_api, addr, grpN); + const account = await updateCachedBalance(addr, grpN); if (account !== null) { console.log( @@ -48,14 +46,4 @@ export async function updateGroupBalances() { } } } - // TODO: move group totatl to env var - // updated CSW_group (note: we have 4 groups as of now) - if (Number(balance_group_to_be_updated) < 4) { - await CURRENT_CSW_GROUP_TO_BE_UPDATED.put( - 'group', - `${Number(balance_group_to_be_updated) + 1}` - ); - } else { - await CURRENT_CSW_GROUP_TO_BE_UPDATED.put('group', `${1}`); - } } diff --git a/src/index.ts b/src/index.ts index 129ee0bb..a0af81f0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,24 +43,3 @@ function registerRoutes(router: Router) { function handleError(error: Error): Response { return new Response(error.message || 'Server Error', { status: 500 }); } - -function getRandomGroup(): number { - const hour = new Date().getHours(); - - if (hour > 0 && hour < 6) { - return 1; - } - - if (hour >= 6 && hour < 12) { - return 2; - } - - if (hour >= 12 && hour < 18) { - return 2; - } - - if (hour >= 18 && hour < 24) { - return 2; - } - throw new Error('invalid hour for group'); -} From 0c7793734b7f878cccc89badaedb7e63ab392aeb Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:42:42 +0300 Subject: [PATCH 066/224] remove CSW KV and change trigger interval for staging --- src/api/bigDipperApi.ts | 2 +- src/bindings.d.ts | 1 - wrangler.toml | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 57f2c7df..91cf04f1 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -5,7 +5,7 @@ import { TotalStakedCoinsResponse, ValidatorDelegationsCountResponse, } from '../types/node'; -import { Account, LatestBlockHeightResponse } from '../types/bigDipper'; +import { Account } from '../types/bigDipper'; import { NodeApi } from './nodeApi'; export class BigDipperApi { diff --git a/src/bindings.d.ts b/src/bindings.d.ts index d0e56a66..32c54676 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -3,7 +3,6 @@ declare global { const REST_API: string; const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; - const CURRENT_CSW_GROUP_TO_BE_UPDATED: KVNamespace; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; } diff --git a/wrangler.toml b/wrangler.toml index 4420db8d..72bc42ed 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -98,12 +98,12 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2"} , { binding = "CURRENT_CSW_GROUP_TO_BE_UPDATED", id = "da8c7da9529943c79167ca1254abc3cc" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2"}} ] # Cron triggers for staging worker [env.staging.triggers] -crons = ["* * * * *"] +crons = ["0 9 * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 2597392cd2e8fae49125dc07cd66182d7d922dc0 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:43:18 +0300 Subject: [PATCH 067/224] remove :delayed --- src/helpers/balanceGroup.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index cfedb54d..79c4e198 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -5,10 +5,6 @@ export function extract_group_number_and_address(key: string) { const parts = key.split(':'); let addr = parts[1]; let grpN = Number(parts[0].split('_')[1]); - - if (key.includes('delayed:')) { - addr = parts[2]; - } return { address: addr, groupNumber: grpN, From 4a884e97ee66a54ade02ec4bec49f1ef11f6b91a Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:53:44 +0300 Subject: [PATCH 068/224] fix wrangler.toml --- src/helpers/balanceGroup.ts | 2 -- wrangler.toml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/helpers/balanceGroup.ts b/src/helpers/balanceGroup.ts index 79c4e198..29974011 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/balanceGroup.ts @@ -12,8 +12,6 @@ export function extract_group_number_and_address(key: string) { } export async function updateGroupBalances(groupNumber: number) { - let node_api = new NodeApi(REST_API); - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${groupNumber}:`, }); diff --git a/wrangler.toml b/wrangler.toml index 72bc42ed..442ee6f8 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -98,7 +98,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2"}} + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2"} ] # Cron triggers for staging worker From 61750fdf5cde777314cc1b89c812ce37ea63f843 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 19 Oct 2022 19:35:21 +0300 Subject: [PATCH 069/224] fix total balance --- src/helpers/balance.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 33910354..a6f1d7c1 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -16,7 +16,7 @@ function extract_account_infos(account: Account) { account?.delegationBalance?.coins && account?.delegationBalance?.coins.length > 0 ) { - delegated = Number(account?.delegationBalance?.coins[0].amount); + delegated = Number(account?.delegationBalance?.coins[0]?.amount || '0'); } let unbonding = 0; @@ -24,13 +24,13 @@ function extract_account_infos(account: Account) { account?.unbondingBalance?.coins && account?.unbondingBalance?.coins.length > 0 ) { - unbonding = Number(account?.unbondingBalance?.coins[0]?.amount); + unbonding = Number(account?.unbondingBalance?.coins[0]?.amount || '0'); } let rewards = 0; if (account?.rewardBalance?.length > 0) { for (let i = 0; i < account?.rewardBalance.length; i++) { - rewards += Number(account?.rewardBalance[i]?.coins[0].amount); + rewards += Number(account?.rewardBalance[i]?.coins[0]?.amount || '0'); } } @@ -49,10 +49,12 @@ export async function get_account_balance_infos( const bd_api = new BigDipperApi(gql_client); const node_api = new NodeApi(REST_API); const latest_block_height = await node_api.get_latest_block_height(); + console.log('height', latest_block_height); const account_balance_infos: Account | null = await bd_api.get_account( address, latest_block_height!! ); + console.log('account infos', account_balance_infos); const { balance, rewards, delegated, unbonding } = extract_account_infos( account_balance_infos!! ); From 0c07fde84edcaa713d1ba6d4d130a58a741f325b Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 20 Oct 2022 14:54:20 +0300 Subject: [PATCH 070/224] decrease height by one --- src/api/nodeApi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index eca22c04..e61d1726 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -57,6 +57,6 @@ export class NodeApi { let respJson = (await resp.json()) as { block: { header: { height: number } }; }; - return Number(respJson.block.header.height); + return Number(respJson.block.header.height) - -1; //TODO: maybe find better way of doin } } From 014f3f37d424e3ecb16a2baca3cb7122d5aef9f9 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 20 Oct 2022 16:05:24 +0300 Subject: [PATCH 071/224] lowwer chances of getting total balance to be 0 --- src/api/nodeApi.ts | 2 +- src/handlers/totalBalance.ts | 13 +++++++++++-- src/helpers/balance.ts | 3 ++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index e61d1726..eca22c04 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -57,6 +57,6 @@ export class NodeApi { let respJson = (await resp.json()) as { block: { header: { height: number } }; }; - return Number(respJson.block.header.height) - -1; //TODO: maybe find better way of doin + return Number(respJson.block.header.height); } } diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 536f5ba9..68422708 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,10 +1,19 @@ import { Request } from 'itty-router'; -import { NodeApi } from '../api/nodeApi'; import { get_account_balance_infos } from '../helpers/balance'; export async function handler(request: Request): Promise { const address = request.params?.['address']; - const account_balance_infos = await get_account_balance_infos(address!!); + let account_balance_infos = await get_account_balance_infos(address!!); + for (let i = 0; i < 5; i++) { + if ( + account_balance_infos?.totalBalance !== 0 && + account_balance_infos?.totalBalance !== undefined + ) { + break; + } + account_balance_infos = await get_account_balance_infos(address!!); + console.log('After trying again...', account_balance_infos?.totalBalance); + } return new Response(account_balance_infos?.totalBalance.toString()); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index a6f1d7c1..8f6de1e6 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -48,8 +48,9 @@ export async function get_account_balance_infos( const gql_client = new GraphQLClient(GRAPHQL_API); const bd_api = new BigDipperApi(gql_client); const node_api = new NodeApi(REST_API); - const latest_block_height = await node_api.get_latest_block_height(); + const latest_block_height = (await node_api.get_latest_block_height()) - 10; console.log('height', latest_block_height); + const account_balance_infos: Account | null = await bd_api.get_account( address, latest_block_height!! From 11e3ffbef231dfd14e092cea38a4fbd23e2f0134 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 21 Oct 2022 00:02:40 -0400 Subject: [PATCH 072/224] Fix typo in available balance --- src/handlers/circulatingSupply.ts | 2 +- src/helpers/balance.ts | 2 +- src/types/node.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 22b4fc10..dd7375cb 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -31,7 +31,7 @@ async function get_circulating_supply(): Promise { if (data !== null) { if ( data.totalBalance === undefined || - data.availiableBalance === undefined || + data.availableBalance === undefined || data.rewards === undefined || data.delegated === undefined || data.unbounding === undefined || diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 8f6de1e6..7b8f68f0 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -63,7 +63,7 @@ export async function get_account_balance_infos( totalBalance: Number( ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding) ), - availiableBalance: balance, + availableBalance: balance, rewards: Number(ncheq_to_cheq_fixed(rewards)), delegated: Number(ncheq_to_cheq_fixed(delegated)), unbounding: Number(ncheq_to_cheq_fixed(unbonding)), diff --git a/src/types/node.ts b/src/types/node.ts index 4edd481b..6afa3d52 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -79,7 +79,7 @@ export interface TotalStakedCoinsResponse { export interface AccountBalanceInfos { totalBalance: number; - availiableBalance: number; + availableBalance: number; rewards: number; delegated: number; unbounding: number; From d8efedfedbf3ebbaeb6d738319fd773c1500396e Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 21 Oct 2022 00:05:19 -0400 Subject: [PATCH 073/224] Fix typo in unbonding balance --- src/handlers/circulatingSupply.ts | 2 +- src/helpers/balance.ts | 2 +- src/types/node.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index dd7375cb..60a0e878 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -34,7 +34,7 @@ async function get_circulating_supply(): Promise { data.availableBalance === undefined || data.rewards === undefined || data.delegated === undefined || - data.unbounding === undefined || + data.unbonding === undefined || data.timeUpdated === undefined ) { const parts = extract_group_number_and_address(key.name); diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 7b8f68f0..86c6a6e3 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -66,7 +66,7 @@ export async function get_account_balance_infos( availableBalance: balance, rewards: Number(ncheq_to_cheq_fixed(rewards)), delegated: Number(ncheq_to_cheq_fixed(delegated)), - unbounding: Number(ncheq_to_cheq_fixed(unbonding)), + unbonding: Number(ncheq_to_cheq_fixed(unbonding)), timeUpdated: new Date().toUTCString(), }; } catch (e) { diff --git a/src/types/node.ts b/src/types/node.ts index 6afa3d52..5bbe4958 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -82,6 +82,6 @@ export interface AccountBalanceInfos { availableBalance: number; rewards: number; delegated: number; - unbounding: number; + unbonding: number; timeUpdated: string; } From cd310b6f79f951359341e5d1c71e10e5fb18a019 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 21 Oct 2022 00:08:00 -0400 Subject: [PATCH 074/224] Convert available balance to CHEQ --- src/helpers/balance.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 86c6a6e3..fe791f77 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -63,11 +63,11 @@ export async function get_account_balance_infos( totalBalance: Number( ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding) ), - availableBalance: balance, + availableBalance: Number(ncheq_to_cheq_fixed(balance)), rewards: Number(ncheq_to_cheq_fixed(rewards)), delegated: Number(ncheq_to_cheq_fixed(delegated)), unbonding: Number(ncheq_to_cheq_fixed(unbonding)), - timeUpdated: new Date().toUTCString(), + timeUpdated: new Date().toUTCString() }; } catch (e) { console.error(e); From 9b024c4e30621d5a733e3bcc16af3074d3cf28a5 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 21 Oct 2022 00:09:09 -0400 Subject: [PATCH 075/224] Update staging trigger to every hour --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 442ee6f8..9d4ab64d 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -103,7 +103,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = ["0 9 * * *"] +crons = ["0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From d4e36ece19c87c5b71d5bcf2af3637afa3f9888a Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 12:57:25 +0300 Subject: [PATCH 076/224] get total delegations balance from node api --- src/api/nodeApi.ts | 20 +++++++++++++++++++- src/helpers/node.ts | 31 ++++++++++++++++++++++++++++++- src/types/node.ts | 20 ++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index eca22c04..54538a96 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -1,4 +1,9 @@ -import { Account, Coin, ValidatorDetailResponse } from '../types/node'; +import { + Account, + Coin, + DelegationsResponse, + ValidatorDetailResponse, +} from '../types/node'; export class NodeApi { constructor(public readonly base_rest_api_url: string) {} @@ -52,6 +57,19 @@ export class NodeApi { return await resp.json(); } + async staking_get_all_delegations_for_delegator( + address: string, + next_key?: string + ) { + const resp = await fetch( + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}${ + next_key ? `?pagination.key=${next_key}` : '' + }` + ); + + return (await resp.json()) as DelegationsResponse; + } + async get_latest_block_height(): Promise { const resp = await fetch(`${this.base_rest_api_url}/blocks/latest`); let respJson = (await resp.json()) as { diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 5419edd5..ce1f9f7c 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,5 +1,6 @@ +import { NodeApi } from '../api/nodeApi'; import { Account } from '../types/bigDipper'; -import { Coin } from '../types/node'; +import { Coin, DelegationsResponse } from '../types/node'; export function total_balance_ncheq(account: Account): number { let balance = Number( @@ -36,3 +37,31 @@ export function total_balance_ncheq(account: Account): number { export function delayed_balance_ncheq(balance: Coin[]): number { return Number(balance.find((c) => c.denom === 'ncheq')?.amount || '0'); } + +export async function calculate_total_delegations_balance_for_delegator( + delegationsResp: DelegationsResponse +): Promise { + let total_delegation_balance = 0; + const next_Key = delegationsResp.pagination.next_key; + + for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { + total_delegation_balance += Number( + delegationsResp.delegation_responses[i].balance.amount + ); + } + + if (next_Key !== null) { + const node_api = new NodeApi(REST_API); + const delegator_address = + delegationsResp.delegation_responses[0].delegation.delegator_address; + + const resp = await node_api.staking_get_all_delegations_for_delegator( + delegator_address, + next_Key + ); + + total_delegation_balance += + await calculate_total_delegations_balance_for_delegator(resp); + } + return total_delegation_balance; +} diff --git a/src/types/node.ts b/src/types/node.ts index 4edd481b..308c1be3 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -85,3 +85,23 @@ export interface AccountBalanceInfos { unbounding: number; timeUpdated: string; } + +export interface DelegationsResponse { + delegation_responses: [ + { + delegation: { + delegator_address: string; + validator_address: string; + shares: string; + }; + balance: { + denom: string; + amount: string; + }; + } + ]; + pagination: { + next_key: string; + total: string; + }; +} From a591b35f4fbf2e37ae547a1a658878814478e788 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 13:27:11 +0300 Subject: [PATCH 077/224] total unbonding balance for delegator from node api --- src/api/nodeApi.ts | 16 +++++++++++++ src/helpers/node.ts | 55 ++++++++++++++++++++++++++++++++++++--------- src/types/node.ts | 21 +++++++++++++++++ 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 54538a96..01cac7f7 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -2,6 +2,7 @@ import { Account, Coin, DelegationsResponse, + UnbondingResponse, ValidatorDetailResponse, } from '../types/node'; @@ -70,6 +71,21 @@ export class NodeApi { return (await resp.json()) as DelegationsResponse; } + async staking_get_all_unboding_delegations_for_delegator( + address: string, + next_key?: string + ) { + const resp = await fetch( + `${ + this.base_rest_api_url + }/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations${ + next_key ? `?pagination.key=${next_key}` : '' + }` + ); + + return (await resp.json()) as UnbondingResponse; + } + async get_latest_block_height(): Promise { const resp = await fetch(`${this.base_rest_api_url}/blocks/latest`); let respJson = (await resp.json()) as { diff --git a/src/helpers/node.ts b/src/helpers/node.ts index ce1f9f7c..088baf1b 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,6 +1,6 @@ import { NodeApi } from '../api/nodeApi'; import { Account } from '../types/bigDipper'; -import { Coin, DelegationsResponse } from '../types/node'; +import { Coin, DelegationsResponse, UnbondingResponse } from '../types/node'; export function total_balance_ncheq(account: Account): number { let balance = Number( @@ -38,30 +38,65 @@ export function delayed_balance_ncheq(balance: Coin[]): number { return Number(balance.find((c) => c.denom === 'ncheq')?.amount || '0'); } -export async function calculate_total_delegations_balance_for_delegator( +export async function calculate_total_delegations_balance_for_delegator_in_ncheq( delegationsResp: DelegationsResponse ): Promise { - let total_delegation_balance = 0; - const next_Key = delegationsResp.pagination.next_key; + let total_delegation_balance_in_ncheq = 0; + const next_key = delegationsResp.pagination.next_key; for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { - total_delegation_balance += Number( + total_delegation_balance_in_ncheq += Number( delegationsResp.delegation_responses[i].balance.amount ); } - if (next_Key !== null) { + if (next_key !== null) { const node_api = new NodeApi(REST_API); const delegator_address = delegationsResp.delegation_responses[0].delegation.delegator_address; const resp = await node_api.staking_get_all_delegations_for_delegator( delegator_address, - next_Key + next_key ); - total_delegation_balance += - await calculate_total_delegations_balance_for_delegator(resp); + total_delegation_balance_in_ncheq += + await calculate_total_delegations_balance_for_delegator_in_ncheq(resp); } - return total_delegation_balance; + + return total_delegation_balance_in_ncheq; +} + +export async function calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( + unbondingResp: UnbondingResponse +): Promise { + let total_unbonding_balance_in_ncheq = 0; + const next_key = unbondingResp.pagination.next_key; + + for (let i = 0; i < unbondingResp.unbonding_responses.length; i++) { + for (let j = 0; j < unbondingResp.unbonding_responses.entries.length; j++) { + total_unbonding_balance_in_ncheq += Number( + unbondingResp.unbonding_responses[i].entries[j].balance + ); + } + } + + if (next_key !== null) { + const node_api = new NodeApi(REST_API); + const delegator_address = + unbondingResp.unbonding_responses[0].delegator_address; + + const resp = + await node_api.staking_get_all_unboding_delegations_for_delegator( + delegator_address, + next_key + ); + + total_unbonding_balance_in_ncheq += + await calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( + resp + ); + } + + return total_unbonding_balance_in_ncheq; } diff --git a/src/types/node.ts b/src/types/node.ts index 308c1be3..f5d786f5 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -105,3 +105,24 @@ export interface DelegationsResponse { total: string; }; } + +export interface UnbondingResponse { + unbonding_responses: [ + { + delegator_address: string; + validator_address: string; + entries: [ + { + creation_height: string; + completion_time: string; + initial_balance: string; + balance: string; + } + ]; + } + ]; + pagination: { + next_key: string; + total: string; + }; +} From 2798e3e3fdd268b432c5e68ce2c0bca855490e79 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:35:55 +0300 Subject: [PATCH 078/224] fix calculate unbonding balance --- src/helpers/node.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 088baf1b..60a0c854 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -74,7 +74,11 @@ export async function calculate_total_unboding_delegations_balance_for_delegator const next_key = unbondingResp.pagination.next_key; for (let i = 0; i < unbondingResp.unbonding_responses.length; i++) { - for (let j = 0; j < unbondingResp.unbonding_responses.entries.length; j++) { + for ( + let j = 0; + j < unbondingResp.unbonding_responses[i].entries.length; + j++ + ) { total_unbonding_balance_in_ncheq += Number( unbondingResp.unbonding_responses[i].entries[j].balance ); From 2ed09b8bfa18db1f1dd41c43ac7720abd59c6fb9 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:41:40 +0300 Subject: [PATCH 079/224] get account balance infos from node for totalBalance --- src/handlers/totalBalance.ts | 17 ++++---------- src/helpers/balance.ts | 44 +++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index 68422708..b8c780ef 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,19 +1,10 @@ import { Request } from 'itty-router'; -import { get_account_balance_infos } from '../helpers/balance'; +import { get_account_balance_infos_from_node_api } from '../helpers/balance'; export async function handler(request: Request): Promise { const address = request.params?.['address']; - let account_balance_infos = await get_account_balance_infos(address!!); - - for (let i = 0; i < 5; i++) { - if ( - account_balance_infos?.totalBalance !== 0 && - account_balance_infos?.totalBalance !== undefined - ) { - break; - } - account_balance_infos = await get_account_balance_infos(address!!); - console.log('After trying again...', account_balance_infos?.totalBalance); - } + let account_balance_infos = await get_account_balance_infos_from_node_api( + address!! + ); return new Response(account_balance_infos?.totalBalance.toString()); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index fe791f77..d8f12043 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -4,6 +4,10 @@ import { Account } from '../types/bigDipper'; import { AccountBalanceInfos } from '../types/node'; import { ncheq_to_cheq_fixed } from './currency'; import { GraphQLClient } from './graphql'; +import { + calculate_total_delegations_balance_for_delegator_in_ncheq, + calculate_total_unboding_delegations_balance_for_delegator_in_ncheq, +} from './node'; function extract_account_infos(account: Account) { let balance = Number( @@ -67,7 +71,7 @@ export async function get_account_balance_infos( rewards: Number(ncheq_to_cheq_fixed(rewards)), delegated: Number(ncheq_to_cheq_fixed(delegated)), unbonding: Number(ncheq_to_cheq_fixed(unbonding)), - timeUpdated: new Date().toUTCString() + timeUpdated: new Date().toUTCString(), }; } catch (e) { console.error(e); @@ -75,6 +79,44 @@ export async function get_account_balance_infos( } } +export async function get_account_balance_infos_from_node_api( + address: string +): Promise { + const node_api = new NodeApi(REST_API); + const available_balance = await node_api.bank_get_account_balances(address); + const available_balance_in_ncheq = Number(available_balance[0].amount); + + const reward_balance_in_ncheq = await node_api.distribution_get_total_rewards( + address + ); + + const total_delegation_balance_in_ncheq = + await calculate_total_delegations_balance_for_delegator_in_ncheq( + await node_api.staking_get_all_delegations_for_delegator(address) + ); + + const total_unbonding_balance_in_ncheq = + await calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( + await node_api.staking_get_all_unboding_delegations_for_delegator(address) + ); + + return { + totalBalance: Number( + ncheq_to_cheq_fixed( + available_balance_in_ncheq + + reward_balance_in_ncheq + + total_delegation_balance_in_ncheq + + total_unbonding_balance_in_ncheq + ) + ), + availableBalance: Number(ncheq_to_cheq_fixed(available_balance_in_ncheq)), + rewards: Number(ncheq_to_cheq_fixed(reward_balance_in_ncheq)), + delegated: Number(ncheq_to_cheq_fixed(total_delegation_balance_in_ncheq)), + unbonding: Number(ncheq_to_cheq_fixed(total_unbonding_balance_in_ncheq)), + timeUpdated: new Date().toUTCString(), + }; +} + export async function updateCachedBalance(addr: string, grpN: number) { try { const account_balance_infos = await get_account_balance_infos(addr); From 89430c26ba3b790d9a9809a3439d01f0d88d8703 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:43:41 +0300 Subject: [PATCH 080/224] update account balance infos from node api --- src/helpers/balance.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index d8f12043..c6914183 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -119,7 +119,9 @@ export async function get_account_balance_infos_from_node_api( export async function updateCachedBalance(addr: string, grpN: number) { try { - const account_balance_infos = await get_account_balance_infos(addr); + const account_balance_infos = await get_account_balance_infos_from_node_api( + addr + ); const data = JSON.stringify(account_balance_infos); From e124c26dfc915972cef93e4ac41012cd3ad0306b Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:57:34 +0300 Subject: [PATCH 081/224] add a logic to calculate vesting and vested(taken form wallet app) --- src/helpers/vesting.ts | 80 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 10 deletions(-) diff --git a/src/helpers/vesting.ts b/src/helpers/vesting.ts index f9c382c0..77dc28ed 100644 --- a/src/helpers/vesting.ts +++ b/src/helpers/vesting.ts @@ -1,24 +1,84 @@ -import { Account } from "../types/node"; +import { Account } from '../types/node'; +import { + is_continuous_vesting_account_type, + is_delayed_vesting_account_type, +} from './validate'; // TODO: This method computes the amount of coins vested. This is not the same as coins that user can spend. // To calculate spendable tokens we need to take into account initial balance + sent and received tokens as well. // Here is the explanation of how to do it properly: // https://docs.cosmos.network/master/modules/auth/05_vesting.html#transferring-sending export function calculate_vested_coins(account: Account): number { - if(account?.["@type"] === "/cosmos.vesting.v1beta1.DelayedVestingAccount" && (Date.now() < account?.base_vesting_account?.end_time * 1000) ) return 0 + if ( + account?.['@type'] === '/cosmos.vesting.v1beta1.DelayedVestingAccount' && + Date.now() < account?.base_vesting_account?.end_time * 1000 + ) + return 0; - const start_time = new Date(account.start_time * 1000).getTime(); - const end_time = new Date(account.base_vesting_account.end_time * 1000).getTime(); - const now = new Date().getTime(); + const start_time = new Date(account.start_time * 1000).getTime(); + const end_time = new Date( + account.base_vesting_account.end_time * 1000 + ).getTime(); + const now = new Date().getTime(); - const time_elapsed = Math.abs(now - start_time) / 1000; - const time_vested = Math.abs(end_time - start_time) / 1000; + const time_elapsed = Math.abs(now - start_time) / 1000; + const time_vested = Math.abs(end_time - start_time) / 1000; - const ratio = Number(time_elapsed / time_vested); + const ratio = Number(time_elapsed / time_vested); - return ratio * Number(account.base_vesting_account.original_vesting[0].amount); + return ( + ratio * Number(account.base_vesting_account.original_vesting[0].amount) + ); } export function calculate_vesting_coins(account: Account): number { - return Number(account.base_vesting_account.original_vesting[0].amount) - calculate_vested_coins(account); + return ( + Number(account.base_vesting_account.original_vesting[0].amount) - + calculate_vested_coins(account) + ); +} + +// Taken from our wallet app +export function estimatedVesting(account: Account, t?: Date) { + if (!t) { + t = new Date(); + } + + if (is_continuous_vesting_account_type(account?.['@type'])) { + const startsAt = account.start_time; + const endsAt = account.base_vesting_account.end_time; + + const totalCoins = Number( + account.base_vesting_account.original_vesting[0]?.amount + ); + + const elapsed = t.getTime() - new Date(startsAt * 1000).getTime(); + const delta = + new Date(endsAt * 1000).getTime() - new Date(startsAt * 1000).getTime(); + + const doneRatio = Math.min(1.0, Math.max(0, elapsed / delta)); + const vested = Math.ceil(Number(totalCoins) * doneRatio); + const vesting = Math.ceil(Number(totalCoins) * (1.0 - doneRatio)); + + return { + vested, + vesting, + }; + } + if (is_delayed_vesting_account_type(account?.['@type'])) { + const endsAt = account.base_vesting_account.end_time; + + const orginalVesting = Number( + account.base_vesting_account.original_vesting[0]?.amount + ); + + const doneRatio = t > new Date(endsAt) ? 1 : 0; + const vested = Math.ceil(Number(orginalVesting) * doneRatio); + const vesting = Math.ceil(Number(orginalVesting) * (1.0 - doneRatio)); + + return { + vested, + vesting, + }; + } } From 160f271a6943e4944ea67a339f93d644d5d248fa Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:58:31 +0300 Subject: [PATCH 082/224] update vesting and vested endpoints to get result from newly added logic --- src/handlers/vestedBalance.ts | 37 +++++++++++++++++++--------------- src/handlers/vestingBalance.ts | 37 +++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index 14d08331..028eb97c 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -1,24 +1,29 @@ -import { Request } from "itty-router"; -import { is_vesting_account_type, validate_cheqd_address } from "../helpers/validate"; -import { NodeApi } from "../api/nodeApi"; -import { calculate_vested_coins } from "../helpers/vesting"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; +import { Request } from 'itty-router'; +import { + is_vesting_account_type, + validate_cheqd_address, +} from '../helpers/validate'; +import { NodeApi } from '../api/nodeApi'; +import { calculate_vested_coins, estimatedVesting } from '../helpers/vesting'; +import { ncheq_to_cheq_fixed } from '../helpers/currency'; export async function handler(request: Request): Promise { - const address = request.params?.['address']; + const address = request.params?.['address']; - if (!address || !validate_cheqd_address(address)) { - throw new Error("No address specified or wrong address format."); - } + if (!address || !validate_cheqd_address(address)) { + throw new Error('No address specified or wrong address format.'); + } - let api = new NodeApi(REST_API); - const account = await api.auth_get_account(address); + let api = new NodeApi(REST_API); + const account = await api.auth_get_account(address); - if (!is_vesting_account_type(account["@type"])) { - throw new Error(`Only vesting accounts are supported. Accounts type '${account["@type"]}'.`) - } + if (!is_vesting_account_type(account['@type'])) { + throw new Error( + `Only vesting accounts are supported. Accounts type '${account['@type']}'.` + ); + } - let vested_coins = calculate_vested_coins(account); + let vested_coins = estimatedVesting(account)?.vested; - return new Response(ncheq_to_cheq_fixed(vested_coins)); + return new Response(ncheq_to_cheq_fixed(vested_coins!!)); } diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index 3e085d3f..10d6ff3f 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -1,24 +1,29 @@ -import { Request } from "itty-router"; -import { is_vesting_account_type, validate_cheqd_address } from "../helpers/validate"; -import { NodeApi } from "../api/nodeApi"; -import { calculate_vesting_coins } from "../helpers/vesting"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; +import { Request } from 'itty-router'; +import { + is_vesting_account_type, + validate_cheqd_address, +} from '../helpers/validate'; +import { NodeApi } from '../api/nodeApi'; +import { calculate_vesting_coins, estimatedVesting } from '../helpers/vesting'; +import { ncheq_to_cheq_fixed } from '../helpers/currency'; export async function handler(request: Request): Promise { - const address = request.params?.['address']; + const address = request.params?.['address']; - if (!address || !validate_cheqd_address(address)) { - throw new Error("No address specified or wrong address format."); - } + if (!address || !validate_cheqd_address(address)) { + throw new Error('No address specified or wrong address format.'); + } - let api = new NodeApi(REST_API); - const account = await api.auth_get_account(address) + let api = new NodeApi(REST_API); + const account = await api.auth_get_account(address); - if (!is_vesting_account_type(account["@type"])) { - throw new Error(`Only vesting accounts are supported. Accounts type '${account["@type"]}'.`) - } + if (!is_vesting_account_type(account['@type'])) { + throw new Error( + `Only vesting accounts are supported. Accounts type '${account['@type']}'.` + ); + } - let vestingCoins = calculate_vesting_coins(account); + let vestingCoins = estimatedVesting(account)?.vesting; - return new Response(ncheq_to_cheq_fixed(vestingCoins)); + return new Response(ncheq_to_cheq_fixed(vestingCoins!!)); } From 284b7466c7d7d2290deb42846822fbbdce84f996 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 21 Oct 2022 16:30:00 +0300 Subject: [PATCH 083/224] review changes --- src/api/bigDipperApi.ts | 2 +- src/handlers/circulatingSupply.ts | 20 ++------------------ src/helpers/balance.ts | 4 ++-- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 91cf04f1..1c4bb259 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -43,7 +43,7 @@ export class BigDipperApi { return resp.data as Account; } catch (e: any) { - console.log(e); + console.log('error get_account: %s', e); return null; } } diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 60a0e878..02cf7d11 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,8 +1,6 @@ import { Request } from 'itty-router'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; -import { updateCachedBalance } from '../helpers/balance'; import { NodeApi } from '../api/nodeApi'; -import { extract_group_number_and_address } from '../helpers/balanceGroup'; import { AccountBalanceInfos } from '../types/node'; async function get_total_supply(): Promise { @@ -28,28 +26,14 @@ async function get_circulating_supply(): Promise { type: 'json', }); - if (data !== null) { - if ( - data.totalBalance === undefined || - data.availableBalance === undefined || - data.rewards === undefined || - data.delegated === undefined || - data.unbonding === undefined || - data.timeUpdated === undefined - ) { - const parts = extract_group_number_and_address(key.name); - updateCachedBalance(parts.address, parts.groupNumber); - } - + if (data !== null && data.totalBalance !== null) { console.log( `found cache entry: ${JSON.stringify(data)} totalBalance=${ data.totalBalance }` ); - if (data.totalBalance !== null) { - shareholders_total_balance += Number(data.totalBalance); - } + shareholders_total_balance += Number(data.totalBalance); } } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index c6914183..2994fb87 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -74,7 +74,7 @@ export async function get_account_balance_infos( timeUpdated: new Date().toUTCString(), }; } catch (e) { - console.error(e); + console.error(`error get_account_balance_infos: ${e}`); return null; } } @@ -129,6 +129,6 @@ export async function updateCachedBalance(addr: string, grpN: number) { console.log(`account "${addr}" balance updated. (${data})`); } catch (e: any) { - console.error('error', e); + console.log(`error updateCachedBalance: ${e}`); } } From b5abbf9f053df7b5f01fa7049d54a961b5637ea0 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 25 Oct 2022 11:02:42 +0300 Subject: [PATCH 084/224] fix a bug when calculating avaliable balance --- src/helpers/balance.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 2994fb87..0c0613b0 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -84,12 +84,15 @@ export async function get_account_balance_infos_from_node_api( ): Promise { const node_api = new NodeApi(REST_API); const available_balance = await node_api.bank_get_account_balances(address); - const available_balance_in_ncheq = Number(available_balance[0].amount); + + let available_balance_in_ncheq = 0; + if (available_balance.length > 0) { + available_balance_in_ncheq = Number(available_balance[0]?.amount); + } const reward_balance_in_ncheq = await node_api.distribution_get_total_rewards( address ); - const total_delegation_balance_in_ncheq = await calculate_total_delegations_balance_for_delegator_in_ncheq( await node_api.staking_get_all_delegations_for_delegator(address) From 0439669b0a4845d22b413381ce1738f74ea316ae Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 25 Oct 2022 11:30:10 +0300 Subject: [PATCH 085/224] remove getting account details from gql --- src/api/bigDipperApi.ts | 37 ------------------------------------- src/helpers/balance.ts | 33 --------------------------------- 2 files changed, 70 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 1c4bb259..b9008e9f 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -11,43 +11,6 @@ import { NodeApi } from './nodeApi'; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} - async get_account(address: string, height: number): Promise { - let query = `query Account($address: String!, $height: Int!) { - accountBalance: action_account_balance(address: $address, height: $height) { - coins - } - delegationBalance: action_delegation_total(address: $address, height: $height) { - coins - } - unbondingBalance: action_unbonding_delegation_total(address: $address, height: $height) { - coins - } - redelegationBalance: action_redelegation(address: $address, height: $height) { - redelegations - } - rewardBalance: action_delegation_reward(address: $address, height: $height) { - coins - } - }`; - - let params = { - address: address, - height: height, - }; - - try { - let resp = await this.graphql_client.query<{ - data: any; - errors: any; - }>(query, params); - - return resp.data as Account; - } catch (e: any) { - console.log('error get_account: %s', e); - return null; - } - } - async get_total_supply(): Promise { let query = `query Supply { supply(order_by: {height:desc} limit: 1) { diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 0c0613b0..844c4f36 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -45,39 +45,6 @@ function extract_account_infos(account: Account) { unbonding, }; } -export async function get_account_balance_infos( - address: string -): Promise { - try { - const gql_client = new GraphQLClient(GRAPHQL_API); - const bd_api = new BigDipperApi(gql_client); - const node_api = new NodeApi(REST_API); - const latest_block_height = (await node_api.get_latest_block_height()) - 10; - console.log('height', latest_block_height); - - const account_balance_infos: Account | null = await bd_api.get_account( - address, - latest_block_height!! - ); - console.log('account infos', account_balance_infos); - const { balance, rewards, delegated, unbonding } = extract_account_infos( - account_balance_infos!! - ); - return { - totalBalance: Number( - ncheq_to_cheq_fixed(balance + rewards + delegated + unbonding) - ), - availableBalance: Number(ncheq_to_cheq_fixed(balance)), - rewards: Number(ncheq_to_cheq_fixed(rewards)), - delegated: Number(ncheq_to_cheq_fixed(delegated)), - unbonding: Number(ncheq_to_cheq_fixed(unbonding)), - timeUpdated: new Date().toUTCString(), - }; - } catch (e) { - console.error(`error get_account_balance_infos: ${e}`); - return null; - } -} export async function get_account_balance_infos_from_node_api( address: string From 997ca12039df9d3091efdbdd0dd64f8bd4f3b370 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 25 Oct 2022 13:43:45 +0300 Subject: [PATCH 086/224] create staging KV for total delegators count --- wrangler.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index 9d4ab64d..96e0022c 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -98,8 +98,8 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2"} -] + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "afb34dd4b9374cc4a7ae7c3aaf6e5ce2"}, { binding = "TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR", id = "adb85e74fe54495481cb41e32e137a21" } +] # Cron triggers for staging worker [env.staging.triggers] From 8db00b91c992abf57890cf7f741683784e526aaf Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 25 Oct 2022 16:29:43 +0300 Subject: [PATCH 087/224] add helper methods to do caching --- src/bindings.d.ts | 1 + src/helpers/constants.ts | 1 + src/helpers/delegatorsCount.ts | 26 ++++++++++++++++++++++++++ src/types/node.ts | 5 +++++ 4 files changed, 33 insertions(+) create mode 100644 src/helpers/delegatorsCount.ts diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 32c54676..6fd28fa8 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -3,6 +3,7 @@ declare global { const REST_API: string; const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; + const TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR: KVNamespace; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; } diff --git a/src/helpers/constants.ts b/src/helpers/constants.ts index 558a1406..c69607c7 100644 --- a/src/helpers/constants.ts +++ b/src/helpers/constants.ts @@ -1 +1,2 @@ export const TOKEN_DECIMALS = 10 ** TOKEN_EXPONENT; +export const LATEST_TOTAL_DELEGATORS_COUNT = 'LATEST_TOTAL_DELEGATORS_COUNT'; diff --git a/src/helpers/delegatorsCount.ts b/src/helpers/delegatorsCount.ts new file mode 100644 index 00000000..dfc40f37 --- /dev/null +++ b/src/helpers/delegatorsCount.ts @@ -0,0 +1,26 @@ +import { CachedTotalDelegatorsCount } from '../types/node'; +import { LATEST_TOTAL_DELEGATORS_COUNT } from './constants'; + +export async function cacheDelegatorsCount(totalDeleagatorsCount: number) { + const data = { + totalDelegatorsCount: totalDeleagatorsCount, + updatedAt: new Date().toUTCString(), + }; + await TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR.put( + LATEST_TOTAL_DELEGATORS_COUNT, + JSON.stringify(data) + ); +} + +export async function getCachedTotalDelegatorsCount(): Promise { + try { + let value = await TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR.get( + LATEST_TOTAL_DELEGATORS_COUNT + ); + + return JSON.parse(value!!) as CachedTotalDelegatorsCount; + } catch (e: any) { + console.error(`getCachedTotalDelegatorsCount: ${e}`); + return null; + } +} diff --git a/src/types/node.ts b/src/types/node.ts index f1b0e440..589cd920 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -126,3 +126,8 @@ export interface UnbondingResponse { total: string; }; } + +export interface CachedTotalDelegatorsCount { + totalDeleagatorsCount: number; + updatedAt: string; +} From 669143e99c68e8e25cd44b5ac6a3e1c6fd793681 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 25 Oct 2022 16:43:38 +0300 Subject: [PATCH 088/224] update total delegators count in cron --- src/handlers/webhookTriggers.ts | 2 ++ .../{delegatorsCount.ts => totalDelegators.ts} | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) rename src/helpers/{delegatorsCount.ts => totalDelegators.ts} (69%) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 4e72a643..73c9c4e2 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,10 +1,12 @@ import { updateGroupBalances } from '../helpers/balanceGroup'; +import { updateTotalDelegatorsCount } from '../helpers/totalDelegators'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); await updateGroupBalances(getRandomGroup()); + await updateTotalDelegatorsCount(); } export async function sendPriceDiscrepancies() { diff --git a/src/helpers/delegatorsCount.ts b/src/helpers/totalDelegators.ts similarity index 69% rename from src/helpers/delegatorsCount.ts rename to src/helpers/totalDelegators.ts index dfc40f37..cffe0a1d 100644 --- a/src/helpers/delegatorsCount.ts +++ b/src/helpers/totalDelegators.ts @@ -1,7 +1,9 @@ +import { BigDipperApi } from '../api/bigDipperApi'; import { CachedTotalDelegatorsCount } from '../types/node'; import { LATEST_TOTAL_DELEGATORS_COUNT } from './constants'; +import { GraphQLClient } from './graphql'; -export async function cacheDelegatorsCount(totalDeleagatorsCount: number) { +export async function cacheDelegatorsCount(totalDeleagatorsCount: Number) { const data = { totalDelegatorsCount: totalDeleagatorsCount, updatedAt: new Date().toUTCString(), @@ -24,3 +26,11 @@ export async function getCachedTotalDelegatorsCount(): Promise Date: Tue, 25 Oct 2022 16:53:11 +0300 Subject: [PATCH 089/224] make total delegators count read from cache if avaliable --- src/handlers/totalDelegators.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/handlers/totalDelegators.ts b/src/handlers/totalDelegators.ts index e6dce728..cb6f6c92 100644 --- a/src/handlers/totalDelegators.ts +++ b/src/handlers/totalDelegators.ts @@ -1,8 +1,18 @@ import { Request } from 'itty-router'; import { BigDipperApi } from '../api/bigDipperApi'; import { GraphQLClient } from '../helpers/graphql'; +import { getCachedTotalDelegatorsCount } from '../helpers/totalDelegators'; export async function handler(request: Request): Promise { + const total_delegators_from_cache = await getCachedTotalDelegatorsCount(); + if ( + total_delegators_from_cache !== null && + total_delegators_from_cache.totalDeleagatorsCount !== undefined + ) { + return new Response( + JSON.stringify(total_delegators_from_cache.totalDeleagatorsCount) + ); + } let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); From 1eb39bbdf1f7ac7994d5abd3375319edc22a896c Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 26 Oct 2022 14:27:32 +0300 Subject: [PATCH 090/224] add preview for CSW KV for local dev --- wrangler.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index f40e618e..4ad7dd78 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -34,6 +34,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa [[kv_namespaces]] binding = "CIRCULATING_SUPPLY_WATCHLIST" id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" +preview_id = "e0229bd428de4499b8a28729c2892a4e" # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` @@ -99,7 +100,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR", id = "adb85e74fe54495481cb41e32e137a21" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR", id = "adb85e74fe54495481cb41e32e137a21" } ] # Cron triggers for staging worker From 3be5b5abfb394a4a1f0a8bbbd4913ae0bc3a21bb Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 26 Oct 2022 14:28:22 +0300 Subject: [PATCH 091/224] switch to node api for getting delegators count per validator --- src/api/nodeApi.ts | 2 +- src/handlers/delegatorCount.ts | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 01cac7f7..db2b7b4e 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -55,7 +55,7 @@ export class NodeApi { `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?pagination.limit=10000` ); - return await resp.json(); + return (await resp.json()) as ValidatorDetailResponse; } async staking_get_all_delegations_for_delegator( diff --git a/src/handlers/delegatorCount.ts b/src/handlers/delegatorCount.ts index 5f18c7f3..9fdc5904 100644 --- a/src/handlers/delegatorCount.ts +++ b/src/handlers/delegatorCount.ts @@ -1,17 +1,17 @@ -import { Request } from "itty-router"; -import { BigDipperApi } from "../api/bigDipperApi"; -import { GraphQLClient } from "../helpers/graphql"; +import { Request } from 'itty-router'; +import { NodeApi } from '../api/nodeApi'; export async function handler(request: Request): Promise { - const address = request.params?.['validator_address']; + const address = request.params?.['validator_address']; - if (!address) { - throw new Error("No address specified or wrong address format."); - } + if (!address) { + throw new Error('No address specified or wrong address format.'); + } - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); + const resp = await new NodeApi(REST_API).staking_get_delegators_per_validator( + address + ); + const delegators_count = resp.delegation_responses.length; - let delegators = await bd_api.get_delegator_count_for_validator(address); - return new Response(delegators.toString()); + return new Response(delegators_count.toString()); } From 2c4a738ff4bd4b19f9c039f94bd0d0783be48684 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 11:10:26 +0300 Subject: [PATCH 092/224] refacor total delegators count a bit --- src/api/bigDipperApi.ts | 32 ++--------------------- src/handlers/totalDelegators.ts | 11 +++++--- src/helpers/totalDelegators.ts | 46 ++++++++++++++++++++++++++++++--- src/types/node.ts | 28 ++++++++++++++++++++ 4 files changed, 81 insertions(+), 36 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index b9008e9f..135bd4de 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -5,9 +5,6 @@ import { TotalStakedCoinsResponse, ValidatorDelegationsCountResponse, } from '../types/node'; -import { Account } from '../types/bigDipper'; -import { NodeApi } from './nodeApi'; - export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} @@ -47,41 +44,16 @@ export class BigDipperApi { return resp.data.delegations.pagination.total; }; - get_total_delegator_count = async (): Promise => { + get_active_validators = async (): Promise => { const queryActiveValidators = `query ActiveValidators { validator_info(distinct_on: operator_address, where: {validator: {validator_statuses: {jailed: {_eq: false}}}}) { operator_address } }`; - - const data = []; - const uniques = new Set(); - const activeValidator = await this.graphql_client.query<{ data: ActiveValidatorsResponse; }>(queryActiveValidators); - - for (let i = 0; i < activeValidator.data.validator_info.length; i++) { - const operator_address = - activeValidator.data.validator_info[i].operator_address; - const resp = await new NodeApi( - REST_API - ).staking_get_delegators_per_validator(operator_address); - data.push({ - validator: operator_address, - delegators: resp.delegation_responses, - }); - } - - for (let i = 0; i < data.length; i++) { - const delegators = data[i].delegators; - for (let j = 0; j < delegators.length; j++) { - uniques.add( - `${delegators[j].delegation.delegator_address}${delegators[j].delegation.validator_address}` - ); - } - } - return uniques.size; + return activeValidator.data; }; get_total_staked_coins = async (): Promise => { diff --git a/src/handlers/totalDelegators.ts b/src/handlers/totalDelegators.ts index cb6f6c92..e3b7c49b 100644 --- a/src/handlers/totalDelegators.ts +++ b/src/handlers/totalDelegators.ts @@ -1,7 +1,10 @@ import { Request } from 'itty-router'; import { BigDipperApi } from '../api/bigDipperApi'; import { GraphQLClient } from '../helpers/graphql'; -import { getCachedTotalDelegatorsCount } from '../helpers/totalDelegators'; +import { + getCachedTotalDelegatorsCount, + getUniqueDelegatorsCountAccrossNetwork, +} from '../helpers/totalDelegators'; export async function handler(request: Request): Promise { const total_delegators_from_cache = await getCachedTotalDelegatorsCount(); @@ -16,6 +19,8 @@ export async function handler(request: Request): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - const delegators = await bd_api.get_total_delegator_count(); - return new Response(JSON.stringify(delegators)); + const active_validators = await bd_api.get_active_validators(); + const total_unique_delegators_count = + await getUniqueDelegatorsCountAccrossNetwork(active_validators); + return new Response(JSON.stringify(total_unique_delegators_count)); } diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index cffe0a1d..a89f87b8 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -1,5 +1,9 @@ import { BigDipperApi } from '../api/bigDipperApi'; -import { CachedTotalDelegatorsCount } from '../types/node'; +import { NodeApi } from '../api/nodeApi'; +import { + ActiveValidatorsResponse, + CachedTotalDelegatorsCount, +} from '../types/node'; import { LATEST_TOTAL_DELEGATORS_COUNT } from './constants'; import { GraphQLClient } from './graphql'; @@ -31,6 +35,42 @@ export async function updateTotalDelegatorsCount() { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - const delegators = await bd_api.get_total_delegator_count(); - await cacheDelegatorsCount(delegators); + const delegators = await bd_api.get_active_validators(); + const delegators_count = await getUniqueDelegatorsCountAccrossNetwork( + delegators + ); + await cacheDelegatorsCount(delegators_count); +} + +export async function getUniqueDelegatorsCountAccrossNetwork( + activeValidators: ActiveValidatorsResponse +): Promise { + const store = []; + const uniques = new Set(); + for (let i = 0; i < activeValidators.validator_info.length; i++) { + const operator_address = + activeValidators.validator_info[i].operator_address; + + const resp = await new NodeApi( + REST_API + ).staking_get_delegators_per_validator(operator_address); + console.log('first operator', resp); + store.push({ + validator: operator_address, + delegators: resp.delegation_responses, + }); + console.log(`At ${i}fetched delegators for validator ${operator_address}`); + } + + console.log('done gettin delegators for each validators'); + + for (let i = 0; i < store.length; i++) { + const delegators = store[i].delegators; + for (let j = 0; j < delegators.length; j++) { + uniques.add( + `${delegators[j].delegation.delegator_address}${delegators[j].delegation.validator_address}` + ); + } + } + return uniques.size; } diff --git a/src/types/node.ts b/src/types/node.ts index 589cd920..fb38000e 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -56,9 +56,18 @@ export interface ValidatorDetailResponse { delegation: { delegator_address: string; validator_address: string; + shares: string; + }; + balance: { + denom: string; + amount: string; }; } ]; + pagination: { + next_key: string; + total: string; + }; } export interface ActiveValidatorsResponse { @@ -106,6 +115,25 @@ export interface DelegationsResponse { }; } +export interface ValidatorDetailResponse { + delegation_responses: [ + { + delegation: { + delegator_address: string; + validator_address: string; + shares: string; + }; + balance: { + denom: string; + amount: string; + }; + } + ]; + pagination: { + next_key: string; + total: string; + }; +} export interface UnbondingResponse { unbonding_responses: [ { From 41b19397bb734270a6e4a4467f0ccb12f45ec54e Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 11:32:32 +0300 Subject: [PATCH 093/224] remove total delegators count kv --- src/bindings.d.ts | 1 - wrangler.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 41474352..5de1c213 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -4,7 +4,6 @@ declare global { const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; const CIRCULATING_SUPPLY_GROUPS: number; - const TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR: KVNamespace; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; } diff --git a/wrangler.toml b/wrangler.toml index 4ad7dd78..cb07abbb 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -100,7 +100,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR", id = "adb85e74fe54495481cb41e32e137a21" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" } ] # Cron triggers for staging worker From ff3791f6437381ad694254ba5cdf194226d3d060 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 11:39:46 +0300 Subject: [PATCH 094/224] setup KV for active validators --- wrangler.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index cb07abbb..b9660196 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -31,10 +31,10 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # KV Namespaces accessible from the Worker # Details: https://developers.cloudflare.com/workers/learning/how-kv-works # @default `[]` -[[kv_namespaces]] -binding = "CIRCULATING_SUPPLY_WATCHLIST" -id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" -preview_id = "e0229bd428de4499b8a28729c2892a4e" + +kv_namespaces = [ + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6", preview_id = "e0229bd428de4499b8a28729c2892a4e" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" } +] # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` @@ -100,7 +100,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1" } ] # Cron triggers for staging worker From 528a26890d9cf8abcba6ba6509d1a760a855c067 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 14:33:27 +0300 Subject: [PATCH 095/224] add a logic to add and remove validators from kv --- src/bindings.d.ts | 1 + src/helpers/totalDelegators.ts | 138 ++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 61 deletions(-) diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 5de1c213..2792729b 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -3,6 +3,7 @@ declare global { const REST_API: string; const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; + const ACTIVE_VALIDATORS: KVNamespace; const CIRCULATING_SUPPLY_GROUPS: number; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index a89f87b8..0768bf90 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -1,76 +1,92 @@ -import { BigDipperApi } from '../api/bigDipperApi'; -import { NodeApi } from '../api/nodeApi'; -import { - ActiveValidatorsResponse, - CachedTotalDelegatorsCount, -} from '../types/node'; -import { LATEST_TOTAL_DELEGATORS_COUNT } from './constants'; -import { GraphQLClient } from './graphql'; +import { ActiveValidatorsResponse } from '../types/node'; +import { extract_group_number_and_address } from './balanceGroup'; -export async function cacheDelegatorsCount(totalDeleagatorsCount: Number) { - const data = { - totalDelegatorsCount: totalDeleagatorsCount, - updatedAt: new Date().toUTCString(), - }; - await TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR.put( - LATEST_TOTAL_DELEGATORS_COUNT, - JSON.stringify(data) - ); -} - -export async function getCachedTotalDelegatorsCount(): Promise { - try { - let value = await TOTAL_DELEGATORS_COUNT_ACROSS_EVERY_VALIDATOR.get( - LATEST_TOTAL_DELEGATORS_COUNT +export async function add_new_active_validators_to_kv( + data: ActiveValidatorsResponse +) { + const latest_active_validators_from_api = data.validator_info; + const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); + const active_validators_from_kv_hashmap = + create_hashmap_of_validators_addresses_from_kv( + active_validators_from_kv.keys as [] ); - return JSON.parse(value!!) as CachedTotalDelegatorsCount; - } catch (e: any) { - console.error(`getCachedTotalDelegatorsCount: ${e}`); - return null; + for (let latest_active_validator of latest_active_validators_from_api) { + // if latest_active_validator is in kv, keep it. + // if latest_active_validator is not in kv, add it. + const is_active_validator_in_kv = + active_validators_from_kv_hashmap.key === + latest_active_validator.operator_address + ? true + : false; + + if (!is_active_validator_in_kv) { + add_new_active_validator_in_kv(latest_active_validator.operator_address); + } } } -export async function updateTotalDelegatorsCount() { - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); +export async function remove_any_jailed_validators_from_kv( + active_validators: ActiveValidatorsResponse +) { + // list of validators form kv + // list of validators from api + // loop throu validators from kv, and if + // a validator from kv doesnt exist in api remove the kv + const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); + const active_validators_from_api_hash_map = + create_hashmap_of_validators_addresses_from_api( + active_validators.validator_info + ); - const delegators = await bd_api.get_active_validators(); - const delegators_count = await getUniqueDelegatorsCountAccrossNetwork( - delegators - ); - await cacheDelegatorsCount(delegators_count); + for (let validator_from_kv of active_validators_from_kv.keys) { + const key_to_look_up = extract_group_number_and_address( + validator_from_kv.name + ).address; + if (active_validators_from_api_hash_map.key !== key_to_look_up) { + delete_stale_validator_from_kv(validator_from_kv.name); + } + } } -export async function getUniqueDelegatorsCountAccrossNetwork( - activeValidators: ActiveValidatorsResponse -): Promise { - const store = []; - const uniques = new Set(); - for (let i = 0; i < activeValidators.validator_info.length; i++) { - const operator_address = - activeValidators.validator_info[i].operator_address; +export function create_hashmap_of_validators_addresses_from_kv( + validators_from_KV: string[] +): { key: string } { + // keys incase contain prexifes, since they are from KV - const resp = await new NodeApi( - REST_API - ).staking_get_delegators_per_validator(operator_address); - console.log('first operator', resp); - store.push({ - validator: operator_address, - delegators: resp.delegation_responses, - }); - console.log(`At ${i}fetched delegators for validator ${operator_address}`); + const hashmap: { key: string } = { key: '' }; + for (let key of validators_from_KV) { + const key_to_look_up = extract_group_number_and_address(key).address; + if (hashmap.key !== key_to_look_up) { + // since kv contains prefix like grp_1.. we need to extract address only + hashmap.key = key_to_look_up; + } } + return hashmap; +} - console.log('done gettin delegators for each validators'); - - for (let i = 0; i < store.length; i++) { - const delegators = store[i].delegators; - for (let j = 0; j < delegators.length; j++) { - uniques.add( - `${delegators[j].delegation.delegator_address}${delegators[j].delegation.validator_address}` - ); +export function create_hashmap_of_validators_addresses_from_api( + validators_from_api: { + operator_address: string; + }[] +): { key: string } { + // keys incase contain prexifes, since they are from KV + const hashmap: { key: string } = { key: '' }; + for (let validator_address of validators_from_api) { + if (hashmap.key !== validator_address.operator_address) { + // since kv contains prefix like grp_1.. we need to extract address only + hashmap.key = validator_address.operator_address; } } - return uniques.size; + return hashmap; +} + +async function add_new_active_validator_in_kv(address: string) { + const data = JSON.stringify({ updatedAt: new Date().toUTCString() }); + // for now manually put em in group 10 + const key = `grp_10:${address}`; + await ACTIVE_VALIDATORS.put(key, data); +} +async function delete_stale_validator_from_kv(key: string) { + await ACTIVE_VALIDATORS.delete(key); } From 3564042f7f94acf702b66b3d34ccac373d7c88c6 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:51:34 +0300 Subject: [PATCH 096/224] set up KV for total delegators --- src/bindings.d.ts | 1 + wrangler.toml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 2792729b..89c26270 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -4,6 +4,7 @@ declare global { const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; const ACTIVE_VALIDATORS: KVNamespace; + const TOTAL_DELEGATORS: KVNamespace; const CIRCULATING_SUPPLY_GROUPS: number; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; diff --git a/wrangler.toml b/wrangler.toml index b9660196..a9145cc5 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -33,7 +33,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6", preview_id = "e0229bd428de4499b8a28729c2892a4e" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6", preview_id = "e0229bd428de4499b8a28729c2892a4e" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" }, { binding = "TOTAL_DELEGATORS", id = "cdb09a8a4c374c5a85b4f532da27fbde" } ] # Map of environment variables to set when deploying the Worker @@ -100,7 +100,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } ] # Cron triggers for staging worker From 2e0eaf84f03336c0b89b01f38af96516dedc4539 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:53:19 +0300 Subject: [PATCH 097/224] paginate and get all delegators for a validator --- src/api/nodeApi.ts | 9 +++++++-- src/helpers/node.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index db2b7b4e..18ae75e5 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -49,10 +49,15 @@ export class NodeApi { } async staking_get_delegators_per_validator( - address: string + address: string, + next_key?: string ): Promise { let resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?pagination.limit=10000` + `${ + this.base_rest_api_url + }/cosmos/staking/v1beta1/validators/${address}/delegations${ + next_key ? `?pagination.key=${next_key}` : '' + }` ); return (await resp.json()) as ValidatorDetailResponse; diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 60a0c854..553f2550 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -104,3 +104,33 @@ export async function calculate_total_unboding_delegations_balance_for_delegator return total_unbonding_balance_in_ncheq; } + +export async function get_all_delegators_for_a_validator( + validator_address: string +): Promise { + const node_api = new NodeApi(REST_API); + let delegationsResp = await node_api.staking_get_delegators_per_validator( + validator_address + ); + let delegators = []; + let next_key = delegationsResp.pagination.next_key; + + while (next_key !== null || delegationsResp.delegation_responses.length > 0) { + for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { + const delegator = + delegationsResp.delegation_responses[i].delegation.delegator_address; + delegators.push(delegator); + } + if (next_key !== null) { + next_key = delegationsResp.pagination.next_key; + delegationsResp = await node_api.staking_get_delegators_per_validator( + validator_address, + next_key + ); + } else { + break; + } + } + + return delegators; +} From 422b321a27388ce994f0105826f593838f175e7b Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:57:28 +0300 Subject: [PATCH 098/224] add a logic to add or remove delegators in KV --- src/helpers/totalDelegators.ts | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index 0768bf90..1f17fb9b 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -1,5 +1,6 @@ import { ActiveValidatorsResponse } from '../types/node'; import { extract_group_number_and_address } from './balanceGroup'; +import { get_all_delegators_for_a_validator } from './node'; export async function add_new_active_validators_to_kv( data: ActiveValidatorsResponse @@ -90,3 +91,45 @@ async function add_new_active_validator_in_kv(address: string) { async function delete_stale_validator_from_kv(key: string) { await ACTIVE_VALIDATORS.delete(key); } + +async function update_delegator_to_validators_KV(validators_group: number) { + const validators = await ACTIVE_VALIDATORS.list({ + prefix: `grp_${validators_group}:`, + }); + + for (let validator of validators.keys) { + const validator_address = extract_group_number_and_address( + validator.name + ).address; + + const delegators_list = await get_all_delegators_for_a_validator( + validator_address + ); + for (let delegator of delegators_list) { + const get_validator_for_delegator_from_kv = (await TOTAL_DELEGATORS.get( + delegator, + { type: 'json' } + )) as string[]; + + if (get_validator_for_delegator_from_kv) { + // delegator has undelegated from all validators + if (get_validator_for_delegator_from_kv.length === 0) { + await TOTAL_DELEGATORS.delete(delegator); + return; + } + // delegator is still delegating, and delegated to new validator + const updated_array = [ + ...get_validator_for_delegator_from_kv, + validator.name, + ]; + + await TOTAL_DELEGATORS.put(delegator, JSON.stringify(updated_array)); + } else { + // delegator is delegating to its first validator + const data = []; + data.push(validator.name); + await TOTAL_DELEGATORS.put(delegator, JSON.stringify(data)); + } + } + } +} From f213e879530cf1a7dd2b4e46c81137f5ff2a6988 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:59:53 +0300 Subject: [PATCH 099/224] change total delegators to read from cache instead --- src/handlers/totalDelegators.ts | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/src/handlers/totalDelegators.ts b/src/handlers/totalDelegators.ts index e3b7c49b..2037e0c8 100644 --- a/src/handlers/totalDelegators.ts +++ b/src/handlers/totalDelegators.ts @@ -1,26 +1,8 @@ import { Request } from 'itty-router'; -import { BigDipperApi } from '../api/bigDipperApi'; -import { GraphQLClient } from '../helpers/graphql'; -import { - getCachedTotalDelegatorsCount, - getUniqueDelegatorsCountAccrossNetwork, -} from '../helpers/totalDelegators'; export async function handler(request: Request): Promise { - const total_delegators_from_cache = await getCachedTotalDelegatorsCount(); - if ( - total_delegators_from_cache !== null && - total_delegators_from_cache.totalDeleagatorsCount !== undefined - ) { - return new Response( - JSON.stringify(total_delegators_from_cache.totalDeleagatorsCount) - ); - } - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); - - const active_validators = await bd_api.get_active_validators(); - const total_unique_delegators_count = - await getUniqueDelegatorsCountAccrossNetwork(active_validators); - return new Response(JSON.stringify(total_unique_delegators_count)); + const total_delegators_from_KV = await ( + await TOTAL_DELEGATORS.list() + ).keys.length; + return new Response(JSON.stringify(total_delegators_from_KV)); } From 39a8685ad8c290964224fc753df0bd6d134dd921 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 18:08:23 +0300 Subject: [PATCH 100/224] set 10 as group for active validators --- src/bindings.d.ts | 1 + wrangler.toml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 89c26270..b55b3808 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -6,6 +6,7 @@ declare global { const ACTIVE_VALIDATORS: KVNamespace; const TOTAL_DELEGATORS: KVNamespace; const CIRCULATING_SUPPLY_GROUPS: number; + const ACTIVE_VALIDATOR_GROUPS: number; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; } diff --git a/wrangler.toml b/wrangler.toml index a9145cc5..11ba3411 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -47,6 +47,8 @@ REST_API = "https://api.cheqd.net" GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" # Number of groups circulating supply watchlist is split into CIRCULATING_SUPPLY_GROUPS = "4" +# Number of groups active validators are split into +ACTIVE_VALIDATOR_GROUPS = "10" # Moniter market API base url MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" @@ -88,8 +90,7 @@ route = { pattern = "data-api-staging.cheqd.io/*", zone_id = "afe3b66243382f2714 # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` -vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net"} - +vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net", ACTIVE_VALIDATOR_GROUPS = "10"} # The necessary secrets are: # - WEBHOOK_URL From 92b388d9a79baef81dc39abce9567c99d55805cf Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 27 Oct 2022 18:09:52 +0300 Subject: [PATCH 101/224] setup a trigger to update delegators to validators --- src/handlers/webhookTriggers.ts | 12 +++++++----- src/helpers/totalDelegators.ts | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index a1bc482b..aee0ae90 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,12 +1,14 @@ import { updateGroupBalances } from '../helpers/balanceGroup'; -import { updateTotalDelegatorsCount } from '../helpers/totalDelegators'; +import { update_delegator_to_validators_KV } from '../helpers/totalDelegators'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); - await updateGroupBalances(getRandomGroup()); - await updateTotalDelegatorsCount(); + await updateGroupBalances(getRandomGroup(CIRCULATING_SUPPLY_GROUPS)); + await update_delegator_to_validators_KV( + getRandomGroup(ACTIVE_VALIDATOR_GROUPS) + ); } export async function sendPriceDiscrepancies() { @@ -34,8 +36,8 @@ export async function sendPriceDiscrepancies() { } } -function getRandomGroup(): number { +function getRandomGroup(group: number): number { let min = 1; - let max = Math.floor(CIRCULATING_SUPPLY_GROUPS); + let max = Math.floor(group); return Math.floor(Math.random() * (max - min + 1)) + min; } diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index 1f17fb9b..a5866209 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -92,7 +92,9 @@ async function delete_stale_validator_from_kv(key: string) { await ACTIVE_VALIDATORS.delete(key); } -async function update_delegator_to_validators_KV(validators_group: number) { +export async function update_delegator_to_validators_KV( + validators_group: number +) { const validators = await ACTIVE_VALIDATORS.list({ prefix: `grp_${validators_group}:`, }); From acbe811d4a60223a2a268823c0b5da1716d4821a Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 28 Oct 2022 10:20:53 +0300 Subject: [PATCH 102/224] add preview id for new KVs --- wrangler.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index 11ba3411..5511a34b 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -33,7 +33,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6", preview_id = "e0229bd428de4499b8a28729c2892a4e" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" }, { binding = "TOTAL_DELEGATORS", id = "cdb09a8a4c374c5a85b4f532da27fbde" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6", preview_id = "e0229bd428de4499b8a28729c2892a4e" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0", preview_id = "ba46538120c645bd9b2056edd95ba250" }, { binding = "TOTAL_DELEGATORS", id = "cdb09a8a4c374c5a85b4f532da27fbde", preview_id = "b1fed5728cb04b2d9008aa38065d19eb" } ] # Map of environment variables to set when deploying the Worker @@ -101,7 +101,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1", preview_id = "5fbc36f8c3c3463586046034bb52117d" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac", preview_id = "94ed81cdb6a444d9923a9fc7497930cc" } ] # Cron triggers for staging worker From e7598f94c4c70c1f76c2efabf297d0530cac1905 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 28 Oct 2022 10:22:19 +0300 Subject: [PATCH 103/224] change delegators count endpoint to use paginated response --- src/handlers/delegatorCount.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/handlers/delegatorCount.ts b/src/handlers/delegatorCount.ts index 9fdc5904..423e5744 100644 --- a/src/handlers/delegatorCount.ts +++ b/src/handlers/delegatorCount.ts @@ -1,5 +1,5 @@ import { Request } from 'itty-router'; -import { NodeApi } from '../api/nodeApi'; +import { get_all_delegators_for_a_validator } from '../helpers/node'; export async function handler(request: Request): Promise { const address = request.params?.['validator_address']; @@ -8,10 +8,8 @@ export async function handler(request: Request): Promise { throw new Error('No address specified or wrong address format.'); } - const resp = await new NodeApi(REST_API).staking_get_delegators_per_validator( - address - ); - const delegators_count = resp.delegation_responses.length; + const resp = await get_all_delegators_for_a_validator(address); + const delegators_count = resp.length; return new Response(delegators_count.toString()); } From 846b48846a1506aa68e56f1971d0615abe6e88cb Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 28 Oct 2022 13:59:32 +0300 Subject: [PATCH 104/224] update next_key after next fetch --- src/helpers/node.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 553f2550..43f5e93c 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -122,11 +122,11 @@ export async function get_all_delegators_for_a_validator( delegators.push(delegator); } if (next_key !== null) { - next_key = delegationsResp.pagination.next_key; delegationsResp = await node_api.staking_get_delegators_per_validator( validator_address, next_key ); + next_key = delegationsResp.pagination.next_key; } else { break; } From 78a4162c94bebd7de67a88c0e7becd0ed32655bd Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 28 Oct 2022 17:51:56 +0300 Subject: [PATCH 105/224] clean up pagination key --- src/api/nodeApi.ts | 4 ++-- src/helpers/node.ts | 30 ++++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 18ae75e5..a803ba1a 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -55,8 +55,8 @@ export class NodeApi { let resp = await fetch( `${ this.base_rest_api_url - }/cosmos/staking/v1beta1/validators/${address}/delegations${ - next_key ? `?pagination.key=${next_key}` : '' + }/cosmos/staking/v1beta1/validators/${address}/delegations?pagination.limit=10${ + next_key ? `&pagination.key=${next_key}` : '' }` ); diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 43f5e93c..af0ef7e1 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -113,7 +113,12 @@ export async function get_all_delegators_for_a_validator( validator_address ); let delegators = []; - let next_key = delegationsResp.pagination.next_key; + let next_key = + delegationsResp.pagination.next_key !== null + ? clean_up_specail_characters_from_pagination_key( + delegationsResp.pagination.next_key + ) + : null; while (next_key !== null || delegationsResp.delegation_responses.length > 0) { for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { @@ -126,11 +131,32 @@ export async function get_all_delegators_for_a_validator( validator_address, next_key ); - next_key = delegationsResp.pagination.next_key; + next_key = + delegationsResp.pagination.next_key !== null + ? clean_up_specail_characters_from_pagination_key( + delegationsResp.pagination.next_key + ) + : null; } else { break; } + console.log(delegationsResp); } return delegators; } + +export function clean_up_specail_characters_from_pagination_key( + unclean_key: string +): string { + const specail_char_1 = '/'; + const specail_char_2 = '+'; + const specail_char_1_placeholder = '%2F'; + const specail_char_2_placeholder = '%2B'; + + return unclean_key + .split(specail_char_1) + .join(specail_char_1_placeholder) + .split(specail_char_2) + .join(specail_char_2_placeholder); +} From 690f9b16a633b2ca48f90d73c9b252eb45b4cc96 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 10:01:29 +0300 Subject: [PATCH 106/224] paginate using offset when getting delegators per validator --- src/api/nodeApi.ts | 19 +++++++---- src/handlers/delegatorCount.ts | 13 +++++--- src/helpers/constants.ts | 2 +- src/helpers/node.ts | 58 ++++++++++++---------------------- 4 files changed, 43 insertions(+), 49 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index a803ba1a..fceeac15 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -1,3 +1,4 @@ +import { PAGINATION_LIMIT } from '../helpers/constants'; import { Account, Coin, @@ -50,14 +51,20 @@ export class NodeApi { async staking_get_delegators_per_validator( address: string, - next_key?: string + offset: number, + should_count_total: boolean ): Promise { + const pagination_count_total = should_count_total + ? 'pagination.count_total=true' + : 'pagination.count_total=false'; + const pagination_limit = `&pagination.limit=${PAGINATION_LIMIT}`; + const pagination_offset = `&pagination.offset=${offset}`; let resp = await fetch( - `${ - this.base_rest_api_url - }/cosmos/staking/v1beta1/validators/${address}/delegations?pagination.limit=10${ - next_key ? `&pagination.key=${next_key}` : '' - }` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations? + ${pagination_count_total} + ${pagination_limit} + ${pagination_offset} + ` ); return (await resp.json()) as ValidatorDetailResponse; diff --git a/src/handlers/delegatorCount.ts b/src/handlers/delegatorCount.ts index 423e5744..1a0bdc6f 100644 --- a/src/handlers/delegatorCount.ts +++ b/src/handlers/delegatorCount.ts @@ -1,5 +1,5 @@ import { Request } from 'itty-router'; -import { get_all_delegators_for_a_validator } from '../helpers/node'; +import { NodeApi } from '../api/nodeApi'; export async function handler(request: Request): Promise { const address = request.params?.['validator_address']; @@ -7,9 +7,14 @@ export async function handler(request: Request): Promise { if (!address) { throw new Error('No address specified or wrong address format.'); } + const node_api = new NodeApi(REST_API); + const resp = await node_api.staking_get_delegators_per_validator( + address, + 0, + true + ); - const resp = await get_all_delegators_for_a_validator(address); - const delegators_count = resp.length; + const total_delegators = resp.pagination.total; - return new Response(delegators_count.toString()); + return new Response(total_delegators.toString()); } diff --git a/src/helpers/constants.ts b/src/helpers/constants.ts index c69607c7..27cb1821 100644 --- a/src/helpers/constants.ts +++ b/src/helpers/constants.ts @@ -1,2 +1,2 @@ export const TOKEN_DECIMALS = 10 ** TOKEN_EXPONENT; -export const LATEST_TOTAL_DELEGATORS_COUNT = 'LATEST_TOTAL_DELEGATORS_COUNT'; +export const PAGINATION_LIMIT = 50; diff --git a/src/helpers/node.ts b/src/helpers/node.ts index af0ef7e1..2e9d8c06 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,6 +1,7 @@ import { NodeApi } from '../api/nodeApi'; import { Account } from '../types/bigDipper'; import { Coin, DelegationsResponse, UnbondingResponse } from '../types/node'; +import { PAGINATION_LIMIT } from './constants'; export function total_balance_ncheq(account: Account): number { let balance = Number( @@ -109,54 +110,35 @@ export async function get_all_delegators_for_a_validator( validator_address: string ): Promise { const node_api = new NodeApi(REST_API); + let offset = 0; + let delegationsResp = await node_api.staking_get_delegators_per_validator( - validator_address + validator_address, + offset, + true // we set it to true to get total_delegators_count ); - let delegators = []; - let next_key = - delegationsResp.pagination.next_key !== null - ? clean_up_specail_characters_from_pagination_key( - delegationsResp.pagination.next_key - ) - : null; - - while (next_key !== null || delegationsResp.delegation_responses.length > 0) { + const total_delegators_count = Number(delegationsResp.pagination.total); + const delegators = []; + + while ( + offset < total_delegators_count || + delegationsResp.delegation_responses.length > 0 + ) { for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { const delegator = delegationsResp.delegation_responses[i].delegation.delegator_address; delegators.push(delegator); } - if (next_key !== null) { - delegationsResp = await node_api.staking_get_delegators_per_validator( - validator_address, - next_key - ); - next_key = - delegationsResp.pagination.next_key !== null - ? clean_up_specail_characters_from_pagination_key( - delegationsResp.pagination.next_key - ) - : null; - } else { + offset += PAGINATION_LIMIT; + delegationsResp = await node_api.staking_get_delegators_per_validator( + validator_address, + offset, + false // we dont need to get total_count on subsequent queries. + ); + if (offset > total_delegators_count) { break; } - console.log(delegationsResp); } return delegators; } - -export function clean_up_specail_characters_from_pagination_key( - unclean_key: string -): string { - const specail_char_1 = '/'; - const specail_char_2 = '+'; - const specail_char_1_placeholder = '%2F'; - const specail_char_2_placeholder = '%2B'; - - return unclean_key - .split(specail_char_1) - .join(specail_char_1_placeholder) - .split(specail_char_2) - .join(specail_char_2_placeholder); -} From 04876f6f1c7b48e960025d62d0aa534b614d8c2d Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 10:28:20 +0300 Subject: [PATCH 107/224] remove newlines and spaces from query --- src/api/nodeApi.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index fceeac15..0d33d15d 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -54,17 +54,15 @@ export class NodeApi { offset: number, should_count_total: boolean ): Promise { + // order of qeury params: count_total -> offset -> limit const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; const pagination_limit = `&pagination.limit=${PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; + // NOTE: be cautios of newlines or spaces. Might malform the request url let resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations? - ${pagination_count_total} - ${pagination_limit} - ${pagination_offset} - ` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}${pagination_offset}${pagination_limit}` ); return (await resp.json()) as ValidatorDetailResponse; From 4edeae9d2d1464eb33d2d9f958787ef20e932328 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 10:52:06 +0300 Subject: [PATCH 108/224] paginate using offeset to get total delegated amount --- src/api/nodeApi.ts | 14 ++++++++++---- src/helpers/balance.ts | 7 ++++++- src/helpers/node.ts | 15 ++++++++++----- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 0d33d15d..c09b34f1 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -70,12 +70,18 @@ export class NodeApi { async staking_get_all_delegations_for_delegator( address: string, - next_key?: string + offset: number, + should_count_total: boolean ) { + // order of qeury params: count_total -> offset -> limit + const pagination_count_total = should_count_total + ? 'pagination.count_total=true' + : 'pagination.count_total=false'; + const pagination_limit = `&pagination.limit=${PAGINATION_LIMIT}`; + const pagination_offset = `&pagination.offset=${offset}`; + // NOTE: be cautios of newlines or spaces. Might malform the request url const resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}${ - next_key ? `?pagination.key=${next_key}` : '' - }` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}${pagination_offset}${pagination_limit}` ); return (await resp.json()) as DelegationsResponse; diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 844c4f36..e5610320 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -62,7 +62,12 @@ export async function get_account_balance_infos_from_node_api( ); const total_delegation_balance_in_ncheq = await calculate_total_delegations_balance_for_delegator_in_ncheq( - await node_api.staking_get_all_delegations_for_delegator(address) + await node_api.staking_get_all_delegations_for_delegator( + address, + 0, + true + ), + 0 ); const total_unbonding_balance_in_ncheq = diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 2e9d8c06..7d442fd1 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -40,10 +40,11 @@ export function delayed_balance_ncheq(balance: Coin[]): number { } export async function calculate_total_delegations_balance_for_delegator_in_ncheq( - delegationsResp: DelegationsResponse + delegationsResp: DelegationsResponse, + current_offset: number ): Promise { let total_delegation_balance_in_ncheq = 0; - const next_key = delegationsResp.pagination.next_key; + const total_count = Number(delegationsResp.pagination.total); for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { total_delegation_balance_in_ncheq += Number( @@ -51,18 +52,22 @@ export async function calculate_total_delegations_balance_for_delegator_in_ncheq ); } - if (next_key !== null) { + if (current_offset < total_count) { const node_api = new NodeApi(REST_API); const delegator_address = delegationsResp.delegation_responses[0].delegation.delegator_address; const resp = await node_api.staking_get_all_delegations_for_delegator( delegator_address, - next_key + current_offset, // our current offset will be updated by recursive call below + true // we count total again , since it's implemented recursively ); total_delegation_balance_in_ncheq += - await calculate_total_delegations_balance_for_delegator_in_ncheq(resp); + await calculate_total_delegations_balance_for_delegator_in_ncheq( + resp, + current_offset + PAGINATION_LIMIT + ); } return total_delegation_balance_in_ncheq; From 702a857d2796b964ca593d7c520dab076de7d8cc Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:06:43 +0300 Subject: [PATCH 109/224] paginate using offest when getting total unboding amount --- src/api/nodeApi.ts | 16 ++++++++++------ src/helpers/balance.ts | 7 ++++++- src/helpers/node.ts | 16 +++++++++------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index c09b34f1..f1c4a530 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -89,14 +89,18 @@ export class NodeApi { async staking_get_all_unboding_delegations_for_delegator( address: string, - next_key?: string + offset: number, + should_count_total: boolean ) { + // order of qeury params: count_total -> offset -> limit + const pagination_count_total = should_count_total + ? 'pagination.count_total=true' + : 'pagination.count_total=false'; + const pagination_limit = `&pagination.limit=${PAGINATION_LIMIT}`; + const pagination_offset = `&pagination.offset=${offset}`; + // NOTE: be cautios of new lines or spaces. Might malform the request url const resp = await fetch( - `${ - this.base_rest_api_url - }/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations${ - next_key ? `?pagination.key=${next_key}` : '' - }` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations?${pagination_count_total}${pagination_offset}${pagination_limit}` ); return (await resp.json()) as UnbondingResponse; diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index e5610320..8d8f5016 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -72,7 +72,12 @@ export async function get_account_balance_infos_from_node_api( const total_unbonding_balance_in_ncheq = await calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( - await node_api.staking_get_all_unboding_delegations_for_delegator(address) + await node_api.staking_get_all_unboding_delegations_for_delegator( + address, + 0, + true + ), + 0 ); return { diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 7d442fd1..8518ecb8 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -74,11 +74,11 @@ export async function calculate_total_delegations_balance_for_delegator_in_ncheq } export async function calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( - unbondingResp: UnbondingResponse + unbondingResp: UnbondingResponse, + current_offset: number ): Promise { let total_unbonding_balance_in_ncheq = 0; - const next_key = unbondingResp.pagination.next_key; - + const total_count = Number(unbondingResp.pagination.total); for (let i = 0; i < unbondingResp.unbonding_responses.length; i++) { for ( let j = 0; @@ -91,7 +91,7 @@ export async function calculate_total_unboding_delegations_balance_for_delegator } } - if (next_key !== null) { + if (current_offset < total_count) { const node_api = new NodeApi(REST_API); const delegator_address = unbondingResp.unbonding_responses[0].delegator_address; @@ -99,12 +99,14 @@ export async function calculate_total_unboding_delegations_balance_for_delegator const resp = await node_api.staking_get_all_unboding_delegations_for_delegator( delegator_address, - next_key + current_offset, + true ); total_unbonding_balance_in_ncheq += await calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( - resp + resp, + current_offset + PAGINATION_LIMIT ); } @@ -138,7 +140,7 @@ export async function get_all_delegators_for_a_validator( delegationsResp = await node_api.staking_get_delegators_per_validator( validator_address, offset, - false // we dont need to get total_count on subsequent queries. + false // we dont need to get total_count on subsequent queries ); if (offset > total_delegators_count) { break; From 9c6e5f80d9f5f46ff88139306c7f40eede020579 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 12:38:19 +0300 Subject: [PATCH 110/224] don't export private functions --- src/helpers/totalDelegators.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index a5866209..ee009d2a 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -50,7 +50,7 @@ export async function remove_any_jailed_validators_from_kv( } } -export function create_hashmap_of_validators_addresses_from_kv( +function create_hashmap_of_validators_addresses_from_kv( validators_from_KV: string[] ): { key: string } { // keys incase contain prexifes, since they are from KV @@ -66,7 +66,7 @@ export function create_hashmap_of_validators_addresses_from_kv( return hashmap; } -export function create_hashmap_of_validators_addresses_from_api( +function create_hashmap_of_validators_addresses_from_api( validators_from_api: { operator_address: string; }[] From 8eb3e7c02e6121f0adacb5e2b466a9b97cdf837e Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 12:47:12 +0300 Subject: [PATCH 111/224] move active validators from param func block --- src/helpers/totalDelegators.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index ee009d2a..9a41b7a8 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -1,10 +1,15 @@ import { ActiveValidatorsResponse } from '../types/node'; import { extract_group_number_and_address } from './balanceGroup'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { GraphQLClient } from './graphql'; import { get_all_delegators_for_a_validator } from './node'; -export async function add_new_active_validators_to_kv( - data: ActiveValidatorsResponse -) { +export async function add_new_active_validators_to_kv() { + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); + const data = await bd_api.get_active_validators() + + const latest_active_validators_from_api = data.validator_info; const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_kv_hashmap = @@ -27,13 +32,15 @@ export async function add_new_active_validators_to_kv( } } -export async function remove_any_jailed_validators_from_kv( - active_validators: ActiveValidatorsResponse -) { +export async function remove_any_jailed_validators_from_kv() { // list of validators form kv // list of validators from api // loop throu validators from kv, and if // a validator from kv doesnt exist in api remove the kv + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); + const active_validators = await bd_api.get_active_validators() + const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_api_hash_map = create_hashmap_of_validators_addresses_from_api( @@ -134,4 +141,4 @@ export async function update_delegator_to_validators_KV( } } } -} + From 4c0d0663ab8135e1572259beabf8e93f8aeb1d5c Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 12:47:39 +0300 Subject: [PATCH 112/224] call add and remove active validators on webtrigger --- src/handlers/webhookTriggers.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index aee0ae90..37ab7e49 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,11 +1,17 @@ import { updateGroupBalances } from '../helpers/balanceGroup'; -import { update_delegator_to_validators_KV } from '../helpers/totalDelegators'; +import { + add_new_active_validators_to_kv, + remove_any_jailed_validators_from_kv, + update_delegator_to_validators_KV, +} from '../helpers/totalDelegators'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); await updateGroupBalances(getRandomGroup(CIRCULATING_SUPPLY_GROUPS)); + await add_new_active_validators_to_kv(); + await remove_any_jailed_validators_from_kv(); await update_delegator_to_validators_KV( getRandomGroup(ACTIVE_VALIDATOR_GROUPS) ); From 2139da5ed3c2b844ae2a48b70444c41bd0ba5a4a Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 12:53:16 +0300 Subject: [PATCH 113/224] fix syntax error --- src/helpers/totalDelegators.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index 9a41b7a8..eb7b42ec 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -1,4 +1,3 @@ -import { ActiveValidatorsResponse } from '../types/node'; import { extract_group_number_and_address } from './balanceGroup'; import { BigDipperApi } from '../api/bigDipperApi'; import { GraphQLClient } from './graphql'; @@ -7,8 +6,7 @@ import { get_all_delegators_for_a_validator } from './node'; export async function add_new_active_validators_to_kv() { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - const data = await bd_api.get_active_validators() - + const data = await bd_api.get_active_validators(); const latest_active_validators_from_api = data.validator_info; const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); @@ -39,7 +37,7 @@ export async function remove_any_jailed_validators_from_kv() { // a validator from kv doesnt exist in api remove the kv let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - const active_validators = await bd_api.get_active_validators() + const active_validators = await bd_api.get_active_validators(); const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_api_hash_map = @@ -141,4 +139,4 @@ export async function update_delegator_to_validators_KV( } } } - +} From abc0845d0c4f79d3eb307890279eed337844df7e Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 17:12:00 +0300 Subject: [PATCH 114/224] fix error at: create_hashmap_of_validators_addresses_from_kv --- src/helpers/totalDelegators.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index eb7b42ec..b677bd08 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -12,7 +12,7 @@ export async function add_new_active_validators_to_kv() { const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_kv_hashmap = create_hashmap_of_validators_addresses_from_kv( - active_validators_from_kv.keys as [] + active_validators_from_kv.keys as KVNamespaceListKey[] ); for (let latest_active_validator of latest_active_validators_from_api) { @@ -56,13 +56,13 @@ export async function remove_any_jailed_validators_from_kv() { } function create_hashmap_of_validators_addresses_from_kv( - validators_from_KV: string[] + validators_from_KV: KVNamespaceListKey[] ): { key: string } { // keys incase contain prexifes, since they are from KV const hashmap: { key: string } = { key: '' }; for (let key of validators_from_KV) { - const key_to_look_up = extract_group_number_and_address(key).address; + const key_to_look_up = extract_group_number_and_address(key.name).address; if (hashmap.key !== key_to_look_up) { // since kv contains prefix like grp_1.. we need to extract address only hashmap.key = key_to_look_up; From 12b8910b207aff9067668217f596a1a1424e04f2 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 14:49:06 +0000 Subject: [PATCH 115/224] Delete get_delegator_count_for_validator --- src/api/bigDipperApi.ts | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 135bd4de..22eb6a2f 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -5,6 +5,7 @@ import { TotalStakedCoinsResponse, ValidatorDelegationsCountResponse, } from '../types/node'; + export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} @@ -23,26 +24,6 @@ export class BigDipperApi { return resp.data.supply[0].coins; } - get_delegator_count_for_validator = async ( - address: string - ): Promise => { - let query = `query ValidatorDelegations($address: String!, $pagination: Boolean! = true) { - delegations: action_validator_delegations(address: $address, count_total: $pagination) { - pagination - } - } - `; - - const params = { - address: address, - }; - - const resp = await this.graphql_client.query<{ - data: ValidatorDelegationsCountResponse; - }>(query, params); - - return resp.data.delegations.pagination.total; - }; get_active_validators = async (): Promise => { const queryActiveValidators = `query ActiveValidators { From 7bb7a09052a9ac95411fc80b6663e6015a6fd55f Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 14:51:03 +0000 Subject: [PATCH 116/224] Rename PAGINATION_LIMIT --- src/api/nodeApi.ts | 14 +++++++------- src/helpers/constants.ts | 2 +- src/helpers/node.ts | 8 ++++---- wrangler.toml | 2 ++ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index f1c4a530..f9a121a7 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -1,4 +1,4 @@ -import { PAGINATION_LIMIT } from '../helpers/constants'; +import { REST_API_PAGINATION_LIMIT } from '../helpers/constants'; import { Account, Coin, @@ -58,11 +58,11 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `&pagination.limit=${PAGINATION_LIMIT}`; + const REST_API_PAGINATION_LIMIT = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; // NOTE: be cautios of newlines or spaces. Might malform the request url let resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}${pagination_offset}${pagination_limit}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}${pagination_offset}${REST_API_PAGINATION_LIMIT}` ); return (await resp.json()) as ValidatorDetailResponse; @@ -77,11 +77,11 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `&pagination.limit=${PAGINATION_LIMIT}`; + const REST_API_PAGINATION_LIMIT = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; // NOTE: be cautios of newlines or spaces. Might malform the request url const resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}${pagination_offset}${pagination_limit}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}${pagination_offset}${REST_API_PAGINATION_LIMIT}` ); return (await resp.json()) as DelegationsResponse; @@ -96,11 +96,11 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `&pagination.limit=${PAGINATION_LIMIT}`; + const REST_API_PAGINATION_LIMIT = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; // NOTE: be cautios of new lines or spaces. Might malform the request url const resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations?${pagination_count_total}${pagination_offset}${pagination_limit}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations?${pagination_count_total}${pagination_offset}${REST_API_PAGINATION_LIMIT}` ); return (await resp.json()) as UnbondingResponse; diff --git a/src/helpers/constants.ts b/src/helpers/constants.ts index 27cb1821..58890396 100644 --- a/src/helpers/constants.ts +++ b/src/helpers/constants.ts @@ -1,2 +1,2 @@ export const TOKEN_DECIMALS = 10 ** TOKEN_EXPONENT; -export const PAGINATION_LIMIT = 50; +export const REST_API_PAGINATION_LIMIT = 50; diff --git a/src/helpers/node.ts b/src/helpers/node.ts index 8518ecb8..d6d99feb 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,7 +1,7 @@ import { NodeApi } from '../api/nodeApi'; import { Account } from '../types/bigDipper'; import { Coin, DelegationsResponse, UnbondingResponse } from '../types/node'; -import { PAGINATION_LIMIT } from './constants'; +import { REST_API_PAGINATION_LIMIT } from './constants'; export function total_balance_ncheq(account: Account): number { let balance = Number( @@ -66,7 +66,7 @@ export async function calculate_total_delegations_balance_for_delegator_in_ncheq total_delegation_balance_in_ncheq += await calculate_total_delegations_balance_for_delegator_in_ncheq( resp, - current_offset + PAGINATION_LIMIT + current_offset + REST_API_PAGINATION_LIMIT ); } @@ -106,7 +106,7 @@ export async function calculate_total_unboding_delegations_balance_for_delegator total_unbonding_balance_in_ncheq += await calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( resp, - current_offset + PAGINATION_LIMIT + current_offset + REST_API_PAGINATION_LIMIT ); } @@ -136,7 +136,7 @@ export async function get_all_delegators_for_a_validator( delegationsResp.delegation_responses[i].delegation.delegator_address; delegators.push(delegator); } - offset += PAGINATION_LIMIT; + offset += REST_API_PAGINATION_LIMIT; delegationsResp = await node_api.staking_get_delegators_per_validator( validator_address, offset, diff --git a/wrangler.toml b/wrangler.toml index 5511a34b..16f9fbd1 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -43,6 +43,8 @@ kv_namespaces = [ TOKEN_EXPONENT = "9" # Standard Cosmosd SDK REST API endpoint for a node on target network REST_API = "https://api.cheqd.net" +# REST API pagination limit +REST_API_PAGINATION_LIMIT = "50" # GraphQL API endpoint for target network. Must be sourced from a BigDipper instance. GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" # Number of groups circulating supply watchlist is split into From 9ca40b616834b4ac80759669666f3767c2cb7e9c Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 14:52:18 +0000 Subject: [PATCH 117/224] Pagination limit binding --- src/bindings.d.ts | 1 + src/helpers/constants.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings.d.ts b/src/bindings.d.ts index b55b3808..0b3a27f5 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -1,6 +1,7 @@ declare global { const TOKEN_EXPONENT: number; const REST_API: string; + const REST_API_PAGINATION_LIMIT: number; const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; const ACTIVE_VALIDATORS: KVNamespace; diff --git a/src/helpers/constants.ts b/src/helpers/constants.ts index 58890396..558a1406 100644 --- a/src/helpers/constants.ts +++ b/src/helpers/constants.ts @@ -1,2 +1 @@ export const TOKEN_DECIMALS = 10 ** TOKEN_EXPONENT; -export const REST_API_PAGINATION_LIMIT = 50; From 883a3fd071c6d78b4e4024c51916b3cab9f5cb74 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 14:58:56 +0000 Subject: [PATCH 118/224] Standardise all node queries --- src/api/nodeApi.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index f9a121a7..d8dc8e80 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -54,15 +54,15 @@ export class NodeApi { offset: number, should_count_total: boolean ): Promise { - // order of qeury params: count_total -> offset -> limit + // order of query params: count_total -> offset -> limit const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const REST_API_PAGINATION_LIMIT = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_limit = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; - // NOTE: be cautios of newlines or spaces. Might malform the request url + // NOTE: be cautious of newlines or spaces. Might make the request URL malformed let resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}${pagination_offset}${REST_API_PAGINATION_LIMIT}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}${pagination_limit}${pagination_offset}` ); return (await resp.json()) as ValidatorDetailResponse; @@ -73,15 +73,15 @@ export class NodeApi { offset: number, should_count_total: boolean ) { - // order of qeury params: count_total -> offset -> limit + // order of query params: count_total -> offset -> limit const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const REST_API_PAGINATION_LIMIT = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_limit = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; - // NOTE: be cautios of newlines or spaces. Might malform the request url + // NOTE: be cautious of newlines or spaces. Might make the request URL malformed const resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}${pagination_offset}${REST_API_PAGINATION_LIMIT}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}${pagination_limit}${pagination_offset}` ); return (await resp.json()) as DelegationsResponse; @@ -92,15 +92,15 @@ export class NodeApi { offset: number, should_count_total: boolean ) { - // order of qeury params: count_total -> offset -> limit + // order of query params: count_total -> offset -> limit const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; const REST_API_PAGINATION_LIMIT = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; - // NOTE: be cautios of new lines or spaces. Might malform the request url + // NOTE: be cautious of new lines or spaces. Might make the request URL malformed const resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations?${pagination_count_total}${pagination_offset}${REST_API_PAGINATION_LIMIT}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations?${pagination_count_total}${pagination_limit}${pagination_offset}` ); return (await resp.json()) as UnbondingResponse; From 29be66782ab7b21742f09d3407cb1ff094d1ff82 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 14:59:51 +0000 Subject: [PATCH 119/224] Correct pagination limit --- src/api/nodeApi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index d8dc8e80..b7ac3c06 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -96,7 +96,7 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const REST_API_PAGINATION_LIMIT = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_limit = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; const pagination_offset = `&pagination.offset=${offset}`; // NOTE: be cautious of new lines or spaces. Might make the request URL malformed const resp = await fetch( From 42cae93267826804ced2d2639d86548b52d4f7b0 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 18:32:12 +0300 Subject: [PATCH 120/224] add logs when updating total delegators --- src/helpers/totalDelegators.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index b677bd08..fa9a607a 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -4,6 +4,7 @@ import { GraphQLClient } from './graphql'; import { get_all_delegators_for_a_validator } from './node'; export async function add_new_active_validators_to_kv() { + console.log('Adding new active validators to KV, if any...'); let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); const data = await bd_api.get_active_validators(); @@ -35,6 +36,7 @@ export async function remove_any_jailed_validators_from_kv() { // list of validators from api // loop throu validators from kv, and if // a validator from kv doesnt exist in api remove the kv + console.log('Removing jailed validators, if any'); let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); const active_validators = await bd_api.get_active_validators(); @@ -58,7 +60,7 @@ export async function remove_any_jailed_validators_from_kv() { function create_hashmap_of_validators_addresses_from_kv( validators_from_KV: KVNamespaceListKey[] ): { key: string } { - // keys incase contain prexifes, since they are from KV + // keys incase contain prefixes, since they are from KV const hashmap: { key: string } = { key: '' }; for (let key of validators_from_KV) { @@ -76,7 +78,7 @@ function create_hashmap_of_validators_addresses_from_api( operator_address: string; }[] ): { key: string } { - // keys incase contain prexifes, since they are from KV + // keys incase contain prefixes, since they are from KV const hashmap: { key: string } = { key: '' }; for (let validator_address of validators_from_api) { if (hashmap.key !== validator_address.operator_address) { @@ -92,18 +94,21 @@ async function add_new_active_validator_in_kv(address: string) { // for now manually put em in group 10 const key = `grp_10:${address}`; await ACTIVE_VALIDATORS.put(key, data); + console.log('Added new validator to the list', address); } async function delete_stale_validator_from_kv(key: string) { await ACTIVE_VALIDATORS.delete(key); + console.log('Deleted stale validator from the list', key); } export async function update_delegator_to_validators_KV( validators_group: number ) { + console.log('Updating total delegator KV...'); const validators = await ACTIVE_VALIDATORS.list({ prefix: `grp_${validators_group}:`, }); - + console.log('updating group ', validators_group); for (let validator of validators.keys) { const validator_address = extract_group_number_and_address( validator.name @@ -122,7 +127,8 @@ export async function update_delegator_to_validators_KV( // delegator has undelegated from all validators if (get_validator_for_delegator_from_kv.length === 0) { await TOTAL_DELEGATORS.delete(delegator); - return; + console.log('Deleted delegator ', delegator); + continue; } // delegator is still delegating, and delegated to new validator const updated_array = [ @@ -131,11 +137,13 @@ export async function update_delegator_to_validators_KV( ]; await TOTAL_DELEGATORS.put(delegator, JSON.stringify(updated_array)); + console.log('Updated delegator: ', delegator); } else { // delegator is delegating to its first validator const data = []; data.push(validator.name); await TOTAL_DELEGATORS.put(delegator, JSON.stringify(data)); + console.log('Added new delegator', delegator); } } } From e4c214404e05673f6118404439b62a1a642f95ab Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 18:51:31 +0300 Subject: [PATCH 121/224] get total supply from gql --- src/api/bigDipperApi.ts | 22 ++++++++++++---------- src/handlers/totalSupply.ts | 11 ++++++----- src/types/bigDipper.ts | 13 +++++++++++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 135bd4de..66d017a5 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,26 +1,28 @@ import { GraphQLClient } from '../helpers/graphql'; +import { TotalSupplyResponse } from '../types/bigDipper'; import { ActiveValidatorsResponse, - Coin, TotalStakedCoinsResponse, ValidatorDelegationsCountResponse, } from '../types/node'; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} - async get_total_supply(): Promise { - let query = `query Supply { - supply(order_by: {height:desc} limit: 1) { - coins - height - } - }`; + async get_total_supply(): Promise { + let query = `query TotalSupply { + supply { + coins + } + }`; let resp = await this.graphql_client.query<{ - data: { supply: { coins: Coin[] }[] }; + data: TotalSupplyResponse; }>(query); - return resp.data.supply[0].coins; + return Number( + resp.data.supply[0].coins.find((coin) => coin.denom === 'ncheq') + ?.amount || '0' + ); } get_delegator_count_for_validator = async ( diff --git a/src/handlers/totalSupply.ts b/src/handlers/totalSupply.ts index 8b15f287..b996aa90 100644 --- a/src/handlers/totalSupply.ts +++ b/src/handlers/totalSupply.ts @@ -1,10 +1,11 @@ import { Request } from 'itty-router'; -import { NodeApi } from '../api/nodeApi'; +import { BigDipperApi } from '../api/bigDipperApi'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { - let nodeApi = new NodeApi(REST_API); - let totalSupply = await nodeApi.bank_get_total_supply_ncheq(); - - return new Response(ncheq_to_cheq_fixed(totalSupply)); + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); + const total_supply = await bd_api.get_total_supply(); + return new Response(ncheq_to_cheq_fixed(total_supply)); } diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index 8c841d54..a8e5215c 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -33,3 +33,16 @@ export class Account { this.vesting_account = vesting_account; } } + +export interface TotalSupplyResponse { + supply: [ + { + coins: [ + { + denom: string; + amount: string; + } + ]; + } + ]; +} From 9b25dfad6b01c8d0495f8a3ea6a056026e89f8a0 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 15:58:41 +0000 Subject: [PATCH 122/224] Remove preview KV pairs --- src/api/nodeApi.ts | 1 - wrangler.toml | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index b7ac3c06..ab4cddf3 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -1,4 +1,3 @@ -import { REST_API_PAGINATION_LIMIT } from '../helpers/constants'; import { Account, Coin, diff --git a/wrangler.toml b/wrangler.toml index 16f9fbd1..c3260364 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -33,7 +33,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6", preview_id = "e0229bd428de4499b8a28729c2892a4e" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0", preview_id = "ba46538120c645bd9b2056edd95ba250" }, { binding = "TOTAL_DELEGATORS", id = "cdb09a8a4c374c5a85b4f532da27fbde", preview_id = "b1fed5728cb04b2d9008aa38065d19eb" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" }, { binding = "TOTAL_DELEGATORS", id = "cdb09a8a4c374c5a85b4f532da27fbde" } ] # Map of environment variables to set when deploying the Worker @@ -92,7 +92,7 @@ route = { pattern = "data-api-staging.cheqd.io/*", zone_id = "afe3b66243382f2714 # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` -vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net", ACTIVE_VALIDATOR_GROUPS = "10"} +vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net", ACTIVE_VALIDATOR_GROUPS = "10" } # The necessary secrets are: # - WEBHOOK_URL @@ -103,7 +103,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab", preview_id = "7330320a78374dab87176419c89e7dcd" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1", preview_id = "5fbc36f8c3c3463586046034bb52117d" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac", preview_id = "94ed81cdb6a444d9923a9fc7497930cc" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } ] # Cron triggers for staging worker From f4dbb115222ecd7717fb54b5f80745bf654cdb01 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 16:44:19 +0000 Subject: [PATCH 123/224] Restructure URL params --- src/api/nodeApi.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index ab4cddf3..1cf52ed3 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -57,11 +57,11 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; - const pagination_offset = `&pagination.offset=${offset}`; + const pagination_limit = `pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_offset = `pagination.offset=${offset}`; // NOTE: be cautious of newlines or spaces. Might make the request URL malformed let resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}${pagination_limit}${pagination_offset}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); return (await resp.json()) as ValidatorDetailResponse; @@ -76,11 +76,11 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; - const pagination_offset = `&pagination.offset=${offset}`; + const pagination_limit = `pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_offset = `pagination.offset=${offset}`; // NOTE: be cautious of newlines or spaces. Might make the request URL malformed const resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}${pagination_limit}${pagination_offset}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); return (await resp.json()) as DelegationsResponse; @@ -95,11 +95,11 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `&pagination.limit=${REST_API_PAGINATION_LIMIT}`; - const pagination_offset = `&pagination.offset=${offset}`; + const pagination_limit = `pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_offset = `pagination.offset=${offset}`; // NOTE: be cautious of new lines or spaces. Might make the request URL malformed const resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations?${pagination_count_total}${pagination_limit}${pagination_offset}` + `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegators/${address}/unbonding_delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); return (await resp.json()) as UnbondingResponse; From 907ce580571ace780e75e010f4890485f0198126 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 16:50:40 +0000 Subject: [PATCH 124/224] Fix missing pagination limit variable in staging --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index c3260364..05a06734 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -92,7 +92,7 @@ route = { pattern = "data-api-staging.cheqd.io/*", zone_id = "afe3b66243382f2714 # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` -vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net", ACTIVE_VALIDATOR_GROUPS = "10" } +vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", REST_API_PAGINATION_LIMIT = "50", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", ACTIVE_VALIDATOR_GROUPS = "10", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net" } # The necessary secrets are: # - WEBHOOK_URL From e5c1eb2aa73124404b8e0c4e155e9ac38fcbc98f Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 16:57:01 +0000 Subject: [PATCH 125/224] Fix typo in unbonding methods --- src/api/nodeApi.ts | 2 +- src/helpers/balance.ts | 6 +++--- src/helpers/node.ts | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 1cf52ed3..ed363f5e 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -86,7 +86,7 @@ export class NodeApi { return (await resp.json()) as DelegationsResponse; } - async staking_get_all_unboding_delegations_for_delegator( + async staking_get_all_unbonding_delegations_for_delegator( address: string, offset: number, should_count_total: boolean diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 8d8f5016..241e8a94 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -6,7 +6,7 @@ import { ncheq_to_cheq_fixed } from './currency'; import { GraphQLClient } from './graphql'; import { calculate_total_delegations_balance_for_delegator_in_ncheq, - calculate_total_unboding_delegations_balance_for_delegator_in_ncheq, + calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq, } from './node'; function extract_account_infos(account: Account) { @@ -71,8 +71,8 @@ export async function get_account_balance_infos_from_node_api( ); const total_unbonding_balance_in_ncheq = - await calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( - await node_api.staking_get_all_unboding_delegations_for_delegator( + await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( + await node_api.staking_get_all_unbonding_delegations_for_delegator( address, 0, true diff --git a/src/helpers/node.ts b/src/helpers/node.ts index d6d99feb..ae7c5b3a 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,7 +1,6 @@ import { NodeApi } from '../api/nodeApi'; import { Account } from '../types/bigDipper'; import { Coin, DelegationsResponse, UnbondingResponse } from '../types/node'; -import { REST_API_PAGINATION_LIMIT } from './constants'; export function total_balance_ncheq(account: Account): number { let balance = Number( @@ -73,7 +72,7 @@ export async function calculate_total_delegations_balance_for_delegator_in_ncheq return total_delegation_balance_in_ncheq; } -export async function calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( +export async function calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( unbondingResp: UnbondingResponse, current_offset: number ): Promise { @@ -97,14 +96,14 @@ export async function calculate_total_unboding_delegations_balance_for_delegator unbondingResp.unbonding_responses[0].delegator_address; const resp = - await node_api.staking_get_all_unboding_delegations_for_delegator( + await node_api.staking_get_all_unbonding_delegations_for_delegator( delegator_address, current_offset, true ); total_unbonding_balance_in_ncheq += - await calculate_total_unboding_delegations_balance_for_delegator_in_ncheq( + await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( resp, current_offset + REST_API_PAGINATION_LIMIT ); From d0e9971cb51d828638986d46d96de2b1bca9b3b1 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 17:00:02 +0000 Subject: [PATCH 126/224] Fix typo in originalVesting --- src/helpers/vesting.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/helpers/vesting.ts b/src/helpers/vesting.ts index 77dc28ed..8964ba60 100644 --- a/src/helpers/vesting.ts +++ b/src/helpers/vesting.ts @@ -68,13 +68,13 @@ export function estimatedVesting(account: Account, t?: Date) { if (is_delayed_vesting_account_type(account?.['@type'])) { const endsAt = account.base_vesting_account.end_time; - const orginalVesting = Number( + const originalVesting = Number( account.base_vesting_account.original_vesting[0]?.amount ); const doneRatio = t > new Date(endsAt) ? 1 : 0; - const vested = Math.ceil(Number(orginalVesting) * doneRatio); - const vesting = Math.ceil(Number(orginalVesting) * (1.0 - doneRatio)); + const vested = Math.ceil(Number(originalVesting) * doneRatio); + const vesting = Math.ceil(Number(originalVesting) * (1.0 - doneRatio)); return { vested, From 3176e7b41347a428120d5f5469a7bba8d2e66536 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 17:04:31 +0000 Subject: [PATCH 127/224] Remove unused delayed vesting functions - Remove marked_as_delayed_vesting_account - Remove filter_marked_as_account_types --- src/helpers/validate.ts | 11 ----------- wrangler.toml | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/helpers/validate.ts b/src/helpers/validate.ts index 58e8878a..8b80fce8 100644 --- a/src/helpers/validate.ts +++ b/src/helpers/validate.ts @@ -14,14 +14,3 @@ export function is_continuous_vesting_account_type(account_type: string): boolea export function is_delayed_vesting_account_type(account_type: string): boolean { return account_type === '/cosmos.vesting.v1beta1.DelayedVestingAccount'; } - -export function marked_as_delayed_vesting_account(address: string): boolean { - return /^delayed:/.test(address); -} - -export function filter_marked_as_account_types(addresses: string[]): Record { - return { - delayed: addresses.filter(address => marked_as_delayed_vesting_account(address)).map(address => address.replace('delayed:', '')), - other: addresses.filter(address => !marked_as_delayed_vesting_account(address)).map(address => address) - }; -} \ No newline at end of file diff --git a/wrangler.toml b/wrangler.toml index 05a06734..fa9acdf8 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -108,7 +108,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = ["0 * * * *"] +crons = ["0/15 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 475521ed30a71071d1e098ecc42fce9a66afab4a Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 17:08:51 +0000 Subject: [PATCH 128/224] Temporarily log request URL for validator delegators --- src/api/nodeApi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index ed363f5e..577b28a3 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -63,7 +63,7 @@ export class NodeApi { let resp = await fetch( `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); - + console.log(resp.url); return (await resp.json()) as ValidatorDetailResponse; } From a2c3061d9d7a2e5aefe55569db730c186f521410 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 20:48:55 +0300 Subject: [PATCH 129/224] fix adding up total delegations and unbondings --- src/helpers/balance.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 241e8a94..a2106df0 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -67,7 +67,7 @@ export async function get_account_balance_infos_from_node_api( 0, true ), - 0 + REST_API_PAGINATION_LIMIT ); const total_unbonding_balance_in_ncheq = @@ -77,7 +77,7 @@ export async function get_account_balance_infos_from_node_api( 0, true ), - 0 + REST_API_PAGINATION_LIMIT ); return { From fe18294befa0c685b1f99a0afcc7a1d9c3445501 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:29:52 +0300 Subject: [PATCH 130/224] use Map instead of object literal --- src/helpers/balance.ts | 8 ++++---- src/helpers/totalDelegators.ts | 27 ++++++++++++++------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index a2106df0..50ca38f6 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -64,20 +64,20 @@ export async function get_account_balance_infos_from_node_api( await calculate_total_delegations_balance_for_delegator_in_ncheq( await node_api.staking_get_all_delegations_for_delegator( address, - 0, + 0, // first call true ), - REST_API_PAGINATION_LIMIT + REST_API_PAGINATION_LIMIT // second call ); const total_unbonding_balance_in_ncheq = await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( await node_api.staking_get_all_unbonding_delegations_for_delegator( address, - 0, + 0, // first call true ), - REST_API_PAGINATION_LIMIT + REST_API_PAGINATION_LIMIT // second call ); return { diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index fa9a607a..05b38b45 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -19,11 +19,9 @@ export async function add_new_active_validators_to_kv() { for (let latest_active_validator of latest_active_validators_from_api) { // if latest_active_validator is in kv, keep it. // if latest_active_validator is not in kv, add it. - const is_active_validator_in_kv = - active_validators_from_kv_hashmap.key === + const is_active_validator_in_kv = active_validators_from_kv_hashmap.has( latest_active_validator.operator_address - ? true - : false; + ); if (!is_active_validator_in_kv) { add_new_active_validator_in_kv(latest_active_validator.operator_address); @@ -51,7 +49,7 @@ export async function remove_any_jailed_validators_from_kv() { const key_to_look_up = extract_group_number_and_address( validator_from_kv.name ).address; - if (active_validators_from_api_hash_map.key !== key_to_look_up) { + if (!active_validators_from_api_hash_map.has(key_to_look_up)) { delete_stale_validator_from_kv(validator_from_kv.name); } } @@ -59,15 +57,15 @@ export async function remove_any_jailed_validators_from_kv() { function create_hashmap_of_validators_addresses_from_kv( validators_from_KV: KVNamespaceListKey[] -): { key: string } { +): Map { // keys incase contain prefixes, since they are from KV - const hashmap: { key: string } = { key: '' }; + const hashmap = new Map(); for (let key of validators_from_KV) { const key_to_look_up = extract_group_number_and_address(key.name).address; - if (hashmap.key !== key_to_look_up) { + if (!hashmap.has(key_to_look_up)) { // since kv contains prefix like grp_1.. we need to extract address only - hashmap.key = key_to_look_up; + hashmap.set(key_to_look_up, key_to_look_up); } } return hashmap; @@ -77,13 +75,16 @@ function create_hashmap_of_validators_addresses_from_api( validators_from_api: { operator_address: string; }[] -): { key: string } { +): Map { // keys incase contain prefixes, since they are from KV - const hashmap: { key: string } = { key: '' }; + const hashmap = new Map(); for (let validator_address of validators_from_api) { - if (hashmap.key !== validator_address.operator_address) { + if (!hashmap.has(validator_address.operator_address)) { // since kv contains prefix like grp_1.. we need to extract address only - hashmap.key = validator_address.operator_address; + hashmap.set( + validator_address.operator_address, + validator_address.operator_address + ); } } return hashmap; From 4f558a13772239242bf66e6493f94299c8ac836d Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Mon, 31 Oct 2022 19:35:30 +0000 Subject: [PATCH 131/224] Reduce staging cron to 1 hour --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index fa9acdf8..05a06734 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -108,7 +108,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = ["0/15 * * * *"] +crons = ["0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 33071f82b09a929183d197204f7b5490b532ae2f Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Tue, 1 Nov 2022 13:40:43 +0300 Subject: [PATCH 132/224] create a new KV for active validators on staging --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index fa9acdf8..c0893737 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -103,7 +103,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "7270ce2a9275432386e59102ee8638b1" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "86a47359834a4ef78d6be7d1bfc83657" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } ] # Cron triggers for staging worker From 70cf0be057f72f63a9c3c985d41a5bf478438013 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 2 Nov 2022 11:45:49 +0300 Subject: [PATCH 133/224] get voting power for active validator --- src/api/bigDipperApi.ts | 12 ++++++++---- src/types/node.ts | 7 +++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 0361db0d..59a20930 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -26,13 +26,17 @@ export class BigDipperApi { ); } - get_active_validators = async (): Promise => { const queryActiveValidators = `query ActiveValidators { - validator_info(distinct_on: operator_address, where: {validator: {validator_statuses: {jailed: {_eq: false}}}}) { - operator_address + validator_info(distinct_on: operator_address, where: {validator: {validator_statuses: {jailed: {_eq: false}}}}) { + operator_address + validator { + validator_voting_powers { + voting_power + } } - }`; + } + }`; const activeValidator = await this.graphql_client.query<{ data: ActiveValidatorsResponse; }>(queryActiveValidators); diff --git a/src/types/node.ts b/src/types/node.ts index fb38000e..e8dc0e99 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -74,6 +74,13 @@ export interface ActiveValidatorsResponse { validator_info: [ { operator_address: string; + validator: { + validator_voting_powers: [ + { + voting_power: number; + } + ]; + }; } ]; } From a22519f5f4554919d40502d23a629039f587044f Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 2 Nov 2022 11:46:26 +0300 Subject: [PATCH 134/224] reuse active validators resp --- src/handlers/webhookTriggers.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 37ab7e49..f29d4601 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,4 +1,6 @@ +import { BigDipperApi } from '../api/bigDipperApi'; import { updateGroupBalances } from '../helpers/balanceGroup'; +import { GraphQLClient } from '../helpers/graphql'; import { add_new_active_validators_to_kv, remove_any_jailed_validators_from_kv, @@ -10,11 +12,16 @@ export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); await updateGroupBalances(getRandomGroup(CIRCULATING_SUPPLY_GROUPS)); - await add_new_active_validators_to_kv(); - await remove_any_jailed_validators_from_kv(); + const gql_client = new GraphQLClient(GRAPHQL_API); + const bd_api = new BigDipperApi(gql_client); + const active_validators_resp = await bd_api.get_active_validators(); + + await remove_any_jailed_validators_from_kv(active_validators_resp); + // also set total delegator count for a validator await update_delegator_to_validators_KV( getRandomGroup(ACTIVE_VALIDATOR_GROUPS) ); + await add_new_active_validators_to_kv(active_validators_resp); } export async function sendPriceDiscrepancies() { From 026f29f7c18ec1d48a3a5193aa2427d1d6c7d132 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 2 Nov 2022 12:04:56 +0300 Subject: [PATCH 135/224] store voting power as well as total delegator count for a validator in KV --- src/helpers/totalDelegators.ts | 100 +++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 30 deletions(-) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index 05b38b45..02022c1a 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -1,15 +1,13 @@ import { extract_group_number_and_address } from './balanceGroup'; -import { BigDipperApi } from '../api/bigDipperApi'; -import { GraphQLClient } from './graphql'; import { get_all_delegators_for_a_validator } from './node'; +import { ActiveValidatorsResponse } from '../types/node'; -export async function add_new_active_validators_to_kv() { +export async function add_new_active_validators_to_kv( + active_validators: ActiveValidatorsResponse +) { console.log('Adding new active validators to KV, if any...'); - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); - const data = await bd_api.get_active_validators(); - const latest_active_validators_from_api = data.validator_info; + const latest_active_validators_from_api = active_validators.validator_info; const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_kv_hashmap = create_hashmap_of_validators_addresses_from_kv( @@ -24,26 +22,29 @@ export async function add_new_active_validators_to_kv() { ); if (!is_active_validator_in_kv) { - add_new_active_validator_in_kv(latest_active_validator.operator_address); + // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. + set_voting_power_for_an_active_validator_in_kv( + latest_active_validator.operator_address, + { + votingPower: + latest_active_validator.validator.validator_voting_powers[0].voting_power.toString(), + } + ); } } } -export async function remove_any_jailed_validators_from_kv() { +export async function remove_any_jailed_validators_from_kv( + active_validators: ActiveValidatorsResponse +) { // list of validators form kv // list of validators from api // loop throu validators from kv, and if // a validator from kv doesnt exist in api remove the kv console.log('Removing jailed validators, if any'); - let gql_client = new GraphQLClient(GRAPHQL_API); - let bd_api = new BigDipperApi(gql_client); - const active_validators = await bd_api.get_active_validators(); - const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_api_hash_map = - create_hashmap_of_validators_addresses_from_api( - active_validators.validator_info - ); + create_hashmap_of_validators_addresses_from_api(active_validators); for (let validator_from_kv of active_validators_from_kv.keys) { const key_to_look_up = extract_group_number_and_address( @@ -72,30 +73,39 @@ function create_hashmap_of_validators_addresses_from_kv( } function create_hashmap_of_validators_addresses_from_api( - validators_from_api: { - operator_address: string; - }[] + validators_from_api: ActiveValidatorsResponse ): Map { // keys incase contain prefixes, since they are from KV const hashmap = new Map(); - for (let validator_address of validators_from_api) { - if (!hashmap.has(validator_address.operator_address)) { + for (let validator of validators_from_api.validator_info) { + if (!hashmap.has(validator.operator_address)) { // since kv contains prefix like grp_1.. we need to extract address only hashmap.set( - validator_address.operator_address, - validator_address.operator_address + validator.operator_address, + validator.validator.validator_voting_powers[0].voting_power.toString() ); } } return hashmap; } -async function add_new_active_validator_in_kv(address: string) { - const data = JSON.stringify({ updatedAt: new Date().toUTCString() }); - // for now manually put em in group 10 - const key = `grp_10:${address}`; - await ACTIVE_VALIDATORS.put(key, data); - console.log('Added new validator to the list', address); +async function set_voting_power_for_an_active_validator_in_kv( + validator_address: string, + data: ActiveValidatorsKV +) { + // get validator from kv first + // set it's voting power + const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { + type: 'json', + })) as ActiveValidatorsKV; + if (validator_from_kv) { + validator_from_kv.votingPower = data.votingPower; + const updated_validator_data = JSON.stringify(validator_from_kv); + // for now manually put em in group 10 + const key = `grp_10:${validator_address}`; + await ACTIVE_VALIDATORS.put(key, updated_validator_data); + console.log('Added new validator to the list', validator_address); + } } async function delete_stale_validator_from_kv(key: string) { await ACTIVE_VALIDATORS.delete(key); @@ -118,11 +128,16 @@ export async function update_delegator_to_validators_KV( const delegators_list = await get_all_delegators_for_a_validator( validator_address ); + const total_delegators_count_for_a_validator = delegators_list.length; + await set_total_delegators_count_for_a_validator( + validator.name, + total_delegators_count_for_a_validator + ); for (let delegator of delegators_list) { const get_validator_for_delegator_from_kv = (await TOTAL_DELEGATORS.get( delegator, { type: 'json' } - )) as string[]; + )) as []; if (get_validator_for_delegator_from_kv) { // delegator has undelegated from all validators @@ -138,6 +153,7 @@ export async function update_delegator_to_validators_KV( ]; await TOTAL_DELEGATORS.put(delegator, JSON.stringify(updated_array)); + console.log('Updated delegator: ', delegator); } else { // delegator is delegating to its first validator @@ -149,3 +165,27 @@ export async function update_delegator_to_validators_KV( } } } + +async function set_total_delegators_count_for_a_validator( + validator_address: string, + total_delegators_count_for_a_validator: number +) { + // get validator from kv first + // set it's total delegators count + const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { + type: 'json', + })) as ActiveValidatorsKV; + if (validator_from_kv) { + validator_from_kv.totalDelegatorsCount = + total_delegators_count_for_a_validator.toString(); + await ACTIVE_VALIDATORS.put( + validator_address, + JSON.stringify(validator_from_kv) + ); + } +} + +export interface ActiveValidatorsKV { + totalDelegatorsCount?: string; + votingPower?: string; +} From fe84c8f6aab83e34a573c868e07184ae268afbd6 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 2 Nov 2022 12:59:10 +0300 Subject: [PATCH 136/224] add new validator to smallest validator group in KV --- src/helpers/totalDelegators.ts | 37 ++++++++++++++++++++++++++++++++-- wrangler.toml | 2 +- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index 02022c1a..f4744599 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -101,8 +101,10 @@ async function set_voting_power_for_an_active_validator_in_kv( if (validator_from_kv) { validator_from_kv.votingPower = data.votingPower; const updated_validator_data = JSON.stringify(validator_from_kv); - // for now manually put em in group 10 - const key = `grp_10:${validator_address}`; + const validator_group_with_smallest_voting_power = + await get_validator_group_with_smallest_voting_power(); + + const key = `grp_${validator_group_with_smallest_voting_power}:${validator_address}`; await ACTIVE_VALIDATORS.put(key, updated_validator_data); console.log('Added new validator to the list', validator_address); } @@ -185,6 +187,37 @@ async function set_total_delegators_count_for_a_validator( } } +async function get_validator_group_with_smallest_voting_power(): Promise { + let voting_power_sum = 0; + let validator_voting_power_total_arr = []; + + for (let i = 1; i <= ACTIVE_VALIDATOR_GROUPS; i++) { + const validator_group = i; + const current_validator_group_data = await ACTIVE_VALIDATORS.list({ + prefix: `grp_${validator_group}:`, + }); + + // accumulates total voting power for specific validator group + for (let validator of current_validator_group_data.keys) { + const validator_data = (await ACTIVE_VALIDATORS.get( + validator.name + )) as ActiveValidatorsKV; + + if (validator_data && validator_data.votingPower) { + voting_power_sum += Number(validator_data.votingPower); + } + } + validator_voting_power_total_arr.push(voting_power_sum); + voting_power_sum = 0; // reset + } + + let smallest_validator_group = 10; // fallback group is 10 + smallest_validator_group = validator_voting_power_total_arr.sort( + (a, b) => a - b + )[0]; // sort by ASC + + return smallest_validator_group; +} export interface ActiveValidatorsKV { totalDelegatorsCount?: string; votingPower?: string; diff --git a/wrangler.toml b/wrangler.toml index c0893737..9c9594f0 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -103,7 +103,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "86a47359834a4ef78d6be7d1bfc83657" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "232b03c1c3784d17b242d4afcd07dec0" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } ] # Cron triggers for staging worker From 796400324b7158f603b0897c408b48f906e152a6 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 2 Nov 2022 13:35:58 +0300 Subject: [PATCH 137/224] try getting delegators count for validator from kv first --- src/handlers/delegatorCount.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/handlers/delegatorCount.ts b/src/handlers/delegatorCount.ts index 1a0bdc6f..a534ad1d 100644 --- a/src/handlers/delegatorCount.ts +++ b/src/handlers/delegatorCount.ts @@ -1,5 +1,6 @@ import { Request } from 'itty-router'; import { NodeApi } from '../api/nodeApi'; +import { ActiveValidatorsKV } from '../helpers/totalDelegators'; export async function handler(request: Request): Promise { const address = request.params?.['validator_address']; @@ -7,6 +8,13 @@ export async function handler(request: Request): Promise { if (!address) { throw new Error('No address specified or wrong address format.'); } + + const total_delegators_from_cache = + await try_getting_delegators_count_from_KV(address); + if (total_delegators_from_cache) { + return new Response(total_delegators_from_cache.toString()); + } + const node_api = new NodeApi(REST_API); const resp = await node_api.staking_get_delegators_per_validator( address, @@ -18,3 +26,13 @@ export async function handler(request: Request): Promise { return new Response(total_delegators.toString()); } + +async function try_getting_delegators_count_from_KV(validator_address: string) { + const validator_data = ACTIVE_VALIDATORS.get( + validator_address + ) as ActiveValidatorsKV; + + return validator_data.totalDelegatorsCount + ? validator_data.totalDelegatorsCount + : null; +} From cde62275fdfcc9557164f5fbcc6abb819b365715 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 2 Nov 2022 13:48:18 +0300 Subject: [PATCH 138/224] cast all numeric constants to Number again. I decided to do this after noticing some requests like this: https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1akqyyul8q32nzxa2yzkenex0q2l4dx6udpjfdf/delegations?pagination.count_total=false&pagination.limit=50&pagination.offset=050 As you can see offset is set to "050", which should be the case of our Numeric constants being are considered as String. I highly doubt this was happing because we're wrapping our Numeric constants in quotes in toml file. --- src/handlers/webhookTriggers.ts | 4 ++-- src/helpers/balance.ts | 4 ++-- src/helpers/node.ts | 6 +++--- src/helpers/totalDelegators.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index f29d4601..274d6bd7 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -11,7 +11,7 @@ import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); - await updateGroupBalances(getRandomGroup(CIRCULATING_SUPPLY_GROUPS)); + await updateGroupBalances(getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS))); const gql_client = new GraphQLClient(GRAPHQL_API); const bd_api = new BigDipperApi(gql_client); const active_validators_resp = await bd_api.get_active_validators(); @@ -19,7 +19,7 @@ export async function webhookTriggers(event: Event) { await remove_any_jailed_validators_from_kv(active_validators_resp); // also set total delegator count for a validator await update_delegator_to_validators_KV( - getRandomGroup(ACTIVE_VALIDATOR_GROUPS) + getRandomGroup(Number(ACTIVE_VALIDATOR_GROUPS)) ); await add_new_active_validators_to_kv(active_validators_resp); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 50ca38f6..682d0220 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -67,7 +67,7 @@ export async function get_account_balance_infos_from_node_api( 0, // first call true ), - REST_API_PAGINATION_LIMIT // second call + Number(REST_API_PAGINATION_LIMIT) // second call ); const total_unbonding_balance_in_ncheq = @@ -77,7 +77,7 @@ export async function get_account_balance_infos_from_node_api( 0, // first call true ), - REST_API_PAGINATION_LIMIT // second call + Number(REST_API_PAGINATION_LIMIT) // second call ); return { diff --git a/src/helpers/node.ts b/src/helpers/node.ts index ae7c5b3a..fcf547dc 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -65,7 +65,7 @@ export async function calculate_total_delegations_balance_for_delegator_in_ncheq total_delegation_balance_in_ncheq += await calculate_total_delegations_balance_for_delegator_in_ncheq( resp, - current_offset + REST_API_PAGINATION_LIMIT + current_offset + Number(REST_API_PAGINATION_LIMIT) ); } @@ -105,7 +105,7 @@ export async function calculate_total_unbonding_delegations_balance_for_delegato total_unbonding_balance_in_ncheq += await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( resp, - current_offset + REST_API_PAGINATION_LIMIT + current_offset + Number(REST_API_PAGINATION_LIMIT) ); } @@ -135,7 +135,7 @@ export async function get_all_delegators_for_a_validator( delegationsResp.delegation_responses[i].delegation.delegator_address; delegators.push(delegator); } - offset += REST_API_PAGINATION_LIMIT; + offset += Number(REST_API_PAGINATION_LIMIT); delegationsResp = await node_api.staking_get_delegators_per_validator( validator_address, offset, diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts index f4744599..0caf9360 100644 --- a/src/helpers/totalDelegators.ts +++ b/src/helpers/totalDelegators.ts @@ -191,7 +191,7 @@ async function get_validator_group_with_smallest_voting_power(): Promise let voting_power_sum = 0; let validator_voting_power_total_arr = []; - for (let i = 1; i <= ACTIVE_VALIDATOR_GROUPS; i++) { + for (let i = 1; i <= Number(ACTIVE_VALIDATOR_GROUPS); i++) { const validator_group = i; const current_validator_group_data = await ACTIVE_VALIDATORS.list({ prefix: `grp_${validator_group}:`, From cc3c9e15bbca6094d08290644233e49254604ade Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 14:12:03 +0000 Subject: [PATCH 139/224] Remove total delegators Remove from KV bindings, helpers, handlers --- src/bindings.d.ts | 1 - src/handlers/totalDelegators.ts | 8 -- src/helpers/totalDelegators.ts | 224 -------------------------------- src/index.ts | 2 - wrangler.toml | 4 +- 5 files changed, 2 insertions(+), 237 deletions(-) delete mode 100644 src/handlers/totalDelegators.ts delete mode 100644 src/helpers/totalDelegators.ts diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 0b3a27f5..9b6e49eb 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -5,7 +5,6 @@ declare global { const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; const ACTIVE_VALIDATORS: KVNamespace; - const TOTAL_DELEGATORS: KVNamespace; const CIRCULATING_SUPPLY_GROUPS: number; const ACTIVE_VALIDATOR_GROUPS: number; const MARKET_MONITORING_API: string; diff --git a/src/handlers/totalDelegators.ts b/src/handlers/totalDelegators.ts deleted file mode 100644 index 2037e0c8..00000000 --- a/src/handlers/totalDelegators.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Request } from 'itty-router'; - -export async function handler(request: Request): Promise { - const total_delegators_from_KV = await ( - await TOTAL_DELEGATORS.list() - ).keys.length; - return new Response(JSON.stringify(total_delegators_from_KV)); -} diff --git a/src/helpers/totalDelegators.ts b/src/helpers/totalDelegators.ts deleted file mode 100644 index 0caf9360..00000000 --- a/src/helpers/totalDelegators.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { extract_group_number_and_address } from './balanceGroup'; -import { get_all_delegators_for_a_validator } from './node'; -import { ActiveValidatorsResponse } from '../types/node'; - -export async function add_new_active_validators_to_kv( - active_validators: ActiveValidatorsResponse -) { - console.log('Adding new active validators to KV, if any...'); - - const latest_active_validators_from_api = active_validators.validator_info; - const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); - const active_validators_from_kv_hashmap = - create_hashmap_of_validators_addresses_from_kv( - active_validators_from_kv.keys as KVNamespaceListKey[] - ); - - for (let latest_active_validator of latest_active_validators_from_api) { - // if latest_active_validator is in kv, keep it. - // if latest_active_validator is not in kv, add it. - const is_active_validator_in_kv = active_validators_from_kv_hashmap.has( - latest_active_validator.operator_address - ); - - if (!is_active_validator_in_kv) { - // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. - set_voting_power_for_an_active_validator_in_kv( - latest_active_validator.operator_address, - { - votingPower: - latest_active_validator.validator.validator_voting_powers[0].voting_power.toString(), - } - ); - } - } -} - -export async function remove_any_jailed_validators_from_kv( - active_validators: ActiveValidatorsResponse -) { - // list of validators form kv - // list of validators from api - // loop throu validators from kv, and if - // a validator from kv doesnt exist in api remove the kv - console.log('Removing jailed validators, if any'); - const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); - const active_validators_from_api_hash_map = - create_hashmap_of_validators_addresses_from_api(active_validators); - - for (let validator_from_kv of active_validators_from_kv.keys) { - const key_to_look_up = extract_group_number_and_address( - validator_from_kv.name - ).address; - if (!active_validators_from_api_hash_map.has(key_to_look_up)) { - delete_stale_validator_from_kv(validator_from_kv.name); - } - } -} - -function create_hashmap_of_validators_addresses_from_kv( - validators_from_KV: KVNamespaceListKey[] -): Map { - // keys incase contain prefixes, since they are from KV - - const hashmap = new Map(); - for (let key of validators_from_KV) { - const key_to_look_up = extract_group_number_and_address(key.name).address; - if (!hashmap.has(key_to_look_up)) { - // since kv contains prefix like grp_1.. we need to extract address only - hashmap.set(key_to_look_up, key_to_look_up); - } - } - return hashmap; -} - -function create_hashmap_of_validators_addresses_from_api( - validators_from_api: ActiveValidatorsResponse -): Map { - // keys incase contain prefixes, since they are from KV - const hashmap = new Map(); - for (let validator of validators_from_api.validator_info) { - if (!hashmap.has(validator.operator_address)) { - // since kv contains prefix like grp_1.. we need to extract address only - hashmap.set( - validator.operator_address, - validator.validator.validator_voting_powers[0].voting_power.toString() - ); - } - } - return hashmap; -} - -async function set_voting_power_for_an_active_validator_in_kv( - validator_address: string, - data: ActiveValidatorsKV -) { - // get validator from kv first - // set it's voting power - const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { - type: 'json', - })) as ActiveValidatorsKV; - if (validator_from_kv) { - validator_from_kv.votingPower = data.votingPower; - const updated_validator_data = JSON.stringify(validator_from_kv); - const validator_group_with_smallest_voting_power = - await get_validator_group_with_smallest_voting_power(); - - const key = `grp_${validator_group_with_smallest_voting_power}:${validator_address}`; - await ACTIVE_VALIDATORS.put(key, updated_validator_data); - console.log('Added new validator to the list', validator_address); - } -} -async function delete_stale_validator_from_kv(key: string) { - await ACTIVE_VALIDATORS.delete(key); - console.log('Deleted stale validator from the list', key); -} - -export async function update_delegator_to_validators_KV( - validators_group: number -) { - console.log('Updating total delegator KV...'); - const validators = await ACTIVE_VALIDATORS.list({ - prefix: `grp_${validators_group}:`, - }); - console.log('updating group ', validators_group); - for (let validator of validators.keys) { - const validator_address = extract_group_number_and_address( - validator.name - ).address; - - const delegators_list = await get_all_delegators_for_a_validator( - validator_address - ); - const total_delegators_count_for_a_validator = delegators_list.length; - await set_total_delegators_count_for_a_validator( - validator.name, - total_delegators_count_for_a_validator - ); - for (let delegator of delegators_list) { - const get_validator_for_delegator_from_kv = (await TOTAL_DELEGATORS.get( - delegator, - { type: 'json' } - )) as []; - - if (get_validator_for_delegator_from_kv) { - // delegator has undelegated from all validators - if (get_validator_for_delegator_from_kv.length === 0) { - await TOTAL_DELEGATORS.delete(delegator); - console.log('Deleted delegator ', delegator); - continue; - } - // delegator is still delegating, and delegated to new validator - const updated_array = [ - ...get_validator_for_delegator_from_kv, - validator.name, - ]; - - await TOTAL_DELEGATORS.put(delegator, JSON.stringify(updated_array)); - - console.log('Updated delegator: ', delegator); - } else { - // delegator is delegating to its first validator - const data = []; - data.push(validator.name); - await TOTAL_DELEGATORS.put(delegator, JSON.stringify(data)); - console.log('Added new delegator', delegator); - } - } - } -} - -async function set_total_delegators_count_for_a_validator( - validator_address: string, - total_delegators_count_for_a_validator: number -) { - // get validator from kv first - // set it's total delegators count - const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { - type: 'json', - })) as ActiveValidatorsKV; - if (validator_from_kv) { - validator_from_kv.totalDelegatorsCount = - total_delegators_count_for_a_validator.toString(); - await ACTIVE_VALIDATORS.put( - validator_address, - JSON.stringify(validator_from_kv) - ); - } -} - -async function get_validator_group_with_smallest_voting_power(): Promise { - let voting_power_sum = 0; - let validator_voting_power_total_arr = []; - - for (let i = 1; i <= Number(ACTIVE_VALIDATOR_GROUPS); i++) { - const validator_group = i; - const current_validator_group_data = await ACTIVE_VALIDATORS.list({ - prefix: `grp_${validator_group}:`, - }); - - // accumulates total voting power for specific validator group - for (let validator of current_validator_group_data.keys) { - const validator_data = (await ACTIVE_VALIDATORS.get( - validator.name - )) as ActiveValidatorsKV; - - if (validator_data && validator_data.votingPower) { - voting_power_sum += Number(validator_data.votingPower); - } - } - validator_voting_power_total_arr.push(voting_power_sum); - voting_power_sum = 0; // reset - } - - let smallest_validator_group = 10; // fallback group is 10 - smallest_validator_group = validator_voting_power_total_arr.sort( - (a, b) => a - b - )[0]; // sort by ASC - - return smallest_validator_group; -} -export interface ActiveValidatorsKV { - totalDelegatorsCount?: string; - votingPower?: string; -} diff --git a/src/index.ts b/src/index.ts index a0af81f0..355880d0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,6 @@ import { handler as liquidBalanceHandler } from './handlers/liquidBalance'; import { handler as vestingBalanceHandler } from './handlers/vestingBalance'; import { handler as vestedBalanceHandler } from './handlers/vestedBalance'; import { handler as delegatorCountHandler } from './handlers/delegatorCount'; -import { handler as totalDelegatorsHandler } from './handlers/totalDelegators'; import { handler as totalStakedCoinsHandler } from './handlers/totalStakedCoins'; import { handler as allArbitrageOpportunitiesHandler } from './handlers/allArbitrageOpportunities'; import { handler as arbitrageOpportunitiesHandler } from './handlers/arbitrageOpportunities'; @@ -30,7 +29,6 @@ function registerRoutes(router: Router) { router.get('/balances/total/:address', totalBalanceHandler); router.get('/balances/vested/:address', vestedBalanceHandler); router.get('/balances/vesting/:address', vestingBalanceHandler); - router.get('/staking/delegators/total', totalDelegatorsHandler); router.get('/staking/delegators/:validator_address', delegatorCountHandler); router.get('/supply/circulating', circulatingSupplyHandler); router.get('/supply/staked', totalStakedCoinsHandler); diff --git a/wrangler.toml b/wrangler.toml index 01ebe8ce..0f57f3a0 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -33,7 +33,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" }, { binding = "TOTAL_DELEGATORS", id = "cdb09a8a4c374c5a85b4f532da27fbde" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" } ] # Map of environment variables to set when deploying the Worker @@ -103,7 +103,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "232b03c1c3784d17b242d4afcd07dec0" }, { binding = "TOTAL_DELEGATORS", id = "596aa5f4f43a4737957a030a2d73a8ac" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "232b03c1c3784d17b242d4afcd07dec0" } ] # Cron triggers for staging worker From 5893d5e7e3fca2053bcb6ee12b4f0f2ac658c733 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 14:15:11 +0000 Subject: [PATCH 140/224] Cleanup BigDipper API and types --- src/api/bigDipperApi.ts | 37 +++++++++++++------------- src/types/bigDipper.ts | 59 ++++++++++++++++------------------------- src/types/node.ts | 23 ---------------- 3 files changed, 41 insertions(+), 78 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 59a20930..27cbe664 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,10 +1,9 @@ import { GraphQLClient } from '../helpers/graphql'; -import { TotalSupplyResponse } from '../types/bigDipper'; -import { - ActiveValidatorsResponse, +import { + TotalSupplyResponse, TotalStakedCoinsResponse, - ValidatorDelegationsCountResponse, -} from '../types/node'; + ActiveValidatorsResponse +} from '../types/bigDipper'; export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} @@ -26,6 +25,19 @@ export class BigDipperApi { ); } + get_total_staked_coins = async (): Promise => { + let query = `query StakingInfo{ + staking_pool { + bonded_tokens + } + }`; + + const resp = await this.graphql_client.query<{ + data: TotalStakedCoinsResponse; + }>(query); + return resp.data.staking_pool[0].bonded_tokens; + } + get_active_validators = async (): Promise => { const queryActiveValidators = `query ActiveValidators { validator_info(distinct_on: operator_address, where: {validator: {validator_statuses: {jailed: {_eq: false}}}}) { @@ -41,18 +53,5 @@ export class BigDipperApi { data: ActiveValidatorsResponse; }>(queryActiveValidators); return activeValidator.data; - }; - - get_total_staked_coins = async (): Promise => { - let query = `query StakingInfo{ - staking_pool { - bonded_tokens - } - }`; - - const resp = await this.graphql_client.query<{ - data: TotalStakedCoinsResponse; - }>(query); - return resp.data.staking_pool[0].bonded_tokens; - }; + } } diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index a8e5215c..17b8f7d9 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -1,39 +1,3 @@ -import { Coin } from './node'; - -export class Account { - public accountBalance: { coins: Coin[] }; - public delegationBalance: { coins: Coin[] }; - public unbondingBalance: { coins: Coin[] }; - public rewardBalance: [{ coins: Coin[] }]; - public vesting_account: { - id: string; - type: string; - original_vesting: Coin[]; - start_time: number; - end_time: number; - }[]; - - constructor( - account_balance: { coins: Coin[] }, - delegation_balance: { coins: Coin[] }, - unbonding_balance: { coins: Coin[] }, - reward_balance: [{ coins: Coin[] }], - vesting_account: { - id: string; - type: string; - original_vesting: Coin[]; - start_time: number; - end_time: number; - }[] - ) { - this.accountBalance = account_balance; - this.delegationBalance = delegation_balance; - this.unbondingBalance = unbonding_balance; - this.rewardBalance = reward_balance; - this.vesting_account = vesting_account; - } -} - export interface TotalSupplyResponse { supply: [ { @@ -46,3 +10,26 @@ export interface TotalSupplyResponse { } ]; } + +export interface TotalStakedCoinsResponse { + staking_pool: [ + { + bonded_tokens: string; + } + ]; +} + +export interface ActiveValidatorsResponse { + validator_info: [ + { + operator_address: string; + validator: { + validator_voting_powers: [ + { + voting_power: number; + } + ]; + }; + } + ]; +} diff --git a/src/types/node.ts b/src/types/node.ts index e8dc0e99..6de137ec 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -70,29 +70,6 @@ export interface ValidatorDetailResponse { }; } -export interface ActiveValidatorsResponse { - validator_info: [ - { - operator_address: string; - validator: { - validator_voting_powers: [ - { - voting_power: number; - } - ]; - }; - } - ]; -} - -export interface TotalStakedCoinsResponse { - staking_pool: [ - { - bonded_tokens: string; - } - ]; -} - export interface AccountBalanceInfos { totalBalance: number; availableBalance: number; From 9f856fd8a7d7ba9dcd6323a056ea6a8106a9f53d Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 14:26:44 +0000 Subject: [PATCH 141/224] Remove extra ValidatorDetailResponse type --- src/types/node.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/types/node.ts b/src/types/node.ts index 6de137ec..7aed257e 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -99,25 +99,6 @@ export interface DelegationsResponse { }; } -export interface ValidatorDetailResponse { - delegation_responses: [ - { - delegation: { - delegator_address: string; - validator_address: string; - shares: string; - }; - balance: { - denom: string; - amount: string; - }; - } - ]; - pagination: { - next_key: string; - total: string; - }; -} export interface UnbondingResponse { unbonding_responses: [ { From 7e4c17cb5f3b84459d166de6739141fc6ceff255 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 14:26:56 +0000 Subject: [PATCH 142/224] Clean up balance helper function --- src/helpers/balance.ts | 56 ------------------------------------------ 1 file changed, 56 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 682d0220..e9d30d33 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,51 +1,11 @@ -import { BigDipperApi } from '../api/bigDipperApi'; import { NodeApi } from '../api/nodeApi'; -import { Account } from '../types/bigDipper'; import { AccountBalanceInfos } from '../types/node'; import { ncheq_to_cheq_fixed } from './currency'; -import { GraphQLClient } from './graphql'; import { calculate_total_delegations_balance_for_delegator_in_ncheq, calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq, } from './node'; -function extract_account_infos(account: Account) { - let balance = Number( - account?.accountBalance?.coins.find((c) => c.denom === 'ncheq')?.amount || - '0' - ); - - let delegated = 0; - if ( - account?.delegationBalance?.coins && - account?.delegationBalance?.coins.length > 0 - ) { - delegated = Number(account?.delegationBalance?.coins[0]?.amount || '0'); - } - - let unbonding = 0; - if ( - account?.unbondingBalance?.coins && - account?.unbondingBalance?.coins.length > 0 - ) { - unbonding = Number(account?.unbondingBalance?.coins[0]?.amount || '0'); - } - - let rewards = 0; - if (account?.rewardBalance?.length > 0) { - for (let i = 0; i < account?.rewardBalance.length; i++) { - rewards += Number(account?.rewardBalance[i]?.coins[0]?.amount || '0'); - } - } - - return { - balance, - rewards, - delegated, - unbonding, - }; -} - export async function get_account_balance_infos_from_node_api( address: string ): Promise { @@ -96,19 +56,3 @@ export async function get_account_balance_infos_from_node_api( timeUpdated: new Date().toUTCString(), }; } - -export async function updateCachedBalance(addr: string, grpN: number) { - try { - const account_balance_infos = await get_account_balance_infos_from_node_api( - addr - ); - - const data = JSON.stringify(account_balance_infos); - - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data); - - console.log(`account "${addr}" balance updated. (${data})`); - } catch (e: any) { - console.log(`error updateCachedBalance: ${e}`); - } -} From 7f23794921bc70a195ca703fbe81fc4be43539f9 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 14:28:41 +0000 Subject: [PATCH 143/224] Rename balanceGroup to circulating --- .../{balanceGroup.ts => circulating.ts} | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) rename src/helpers/{balanceGroup.ts => circulating.ts} (67%) diff --git a/src/helpers/balanceGroup.ts b/src/helpers/circulating.ts similarity index 67% rename from src/helpers/balanceGroup.ts rename to src/helpers/circulating.ts index ed71da2b..815bdc44 100644 --- a/src/helpers/balanceGroup.ts +++ b/src/helpers/circulating.ts @@ -1,4 +1,4 @@ -import { updateCachedBalance } from './balance'; +import { get_account_balance_infos_from_node_api } from './balance'; export function extract_group_number_and_address(key: string) { const parts = key.split(':'); @@ -40,3 +40,19 @@ export async function updateGroupBalances(groupNumber: number) { } } } + +export async function updateCachedBalance(addr: string, grpN: number) { + try { + const account_balance_infos = await get_account_balance_infos_from_node_api( + addr + ); + + const data = JSON.stringify(account_balance_infos); + + await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data); + + console.log(`account "${addr}" balance updated. (${data})`); + } catch (e: any) { + console.log(`error updateCachedBalance: ${e}`); + } +} From 97f7eb38289836a40d75888ba61ce7fb54f6eefe Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 15:04:44 +0000 Subject: [PATCH 144/224] Add RewardsResponse type --- src/api/nodeApi.ts | 5 +---- src/types/node.ts | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 577b28a3..bc29de3d 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -40,10 +40,7 @@ export class NodeApi { let resp = await fetch( `${this.base_rest_api_url}/cosmos/distribution/v1beta1/delegators/${address}/rewards` ); - let respJson = (await resp.json()) as { - rewards: Record[]; - total: Coin[]; - }; + let respJson = (await resp.json()) as RewardsResponse return Number(respJson?.total?.[0]?.amount ?? '0'); } diff --git a/src/types/node.ts b/src/types/node.ts index 7aed257e..59225091 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -120,7 +120,7 @@ export interface UnbondingResponse { }; } -export interface CachedTotalDelegatorsCount { - totalDeleagatorsCount: number; - updatedAt: string; +export interface RewardsResponse { + rewards: Record[]; + total: Coin[]; } From a2f2b5fb778d29ec4d7805f0e598c53c87c99c2c Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 15:05:06 +0000 Subject: [PATCH 145/224] Rename circulating supply helper --- src/handlers/webhookTriggers.ts | 14 +-- src/helpers/circulating.ts | 2 +- src/helpers/validators.ts | 171 ++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 src/helpers/validators.ts diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 274d6bd7..78caa611 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,17 +1,13 @@ -import { BigDipperApi } from '../api/bigDipperApi'; -import { updateGroupBalances } from '../helpers/balanceGroup'; -import { GraphQLClient } from '../helpers/graphql'; -import { - add_new_active_validators_to_kv, - remove_any_jailed_validators_from_kv, - update_delegator_to_validators_KV, -} from '../helpers/totalDelegators'; +import { updateCirculatingSupply } from '../helpers/circulating'; +import { updateValidatorKV } from '../helpers/validators'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); - await updateGroupBalances(getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS))); + await updateCirculatingSupply(getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS))); + await de + const gql_client = new GraphQLClient(GRAPHQL_API); const bd_api = new BigDipperApi(gql_client); const active_validators_resp = await bd_api.get_active_validators(); diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 815bdc44..1ed4f4ae 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -10,7 +10,7 @@ export function extract_group_number_and_address(key: string) { }; } -export async function updateGroupBalances(groupNumber: number) { +export async function updateCirculatingSupply(groupNumber: number) { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ prefix: `grp_${groupNumber}:`, }); diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts new file mode 100644 index 00000000..68ab327b --- /dev/null +++ b/src/helpers/validators.ts @@ -0,0 +1,171 @@ +import { extract_group_number_and_address } from './circulating'; +import { get_all_delegators_for_a_validator } from './node'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { ActiveValidatorsResponse } from '../types/bigDipper'; + +export async function add_new_active_validators_to_kv( + active_validators: ActiveValidatorsResponse +) { + console.log('Adding new active validators to KV, if any...'); + + const latest_active_validators_from_api = active_validators.validator_info; + const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); + const active_validators_from_kv_hashmap = + create_hashmap_of_validators_addresses_from_kv( + active_validators_from_kv.keys as KVNamespaceListKey[] + ); + + for (let latest_active_validator of latest_active_validators_from_api) { + // if latest_active_validator is in kv, keep it. + // if latest_active_validator is not in kv, add it. + const is_active_validator_in_kv = active_validators_from_kv_hashmap.has( + latest_active_validator.operator_address + ); + + if (!is_active_validator_in_kv) { + // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. + set_voting_power_for_an_active_validator_in_kv( + latest_active_validator.operator_address, + { + votingPower: + latest_active_validator.validator.validator_voting_powers[0].voting_power.toString(), + } + ); + } + } +} + +export async function remove_any_jailed_validators_from_kv( + active_validators: ActiveValidatorsResponse +) { + // list of validators form kv + // list of validators from api + // loop throu validators from kv, and if + // a validator from kv doesnt exist in api remove the kv + console.log('Removing jailed validators, if any'); + const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); + const active_validators_from_api_hash_map = + create_hashmap_of_validators_addresses_from_api(active_validators); + + for (let validator_from_kv of active_validators_from_kv.keys) { + const key_to_look_up = extract_group_number_and_address( + validator_from_kv.name + ).address; + if (!active_validators_from_api_hash_map.has(key_to_look_up)) { + delete_stale_validator_from_kv(validator_from_kv.name); + } + } +} + +function create_hashmap_of_validators_addresses_from_kv( + validators_from_KV: KVNamespaceListKey[] +): Map { + // keys incase contain prefixes, since they are from KV + + const hashmap = new Map(); + for (let key of validators_from_KV) { + const key_to_look_up = extract_group_number_and_address(key.name).address; + if (!hashmap.has(key_to_look_up)) { + // since kv contains prefix like grp_1.. we need to extract address only + hashmap.set(key_to_look_up, key_to_look_up); + } + } + return hashmap; +} + +function create_hashmap_of_validators_addresses_from_api( + validators_from_api: ActiveValidatorsResponse +): Map { + // keys incase contain prefixes, since they are from KV + const hashmap = new Map(); + for (let validator of validators_from_api.validator_info) { + if (!hashmap.has(validator.operator_address)) { + // since kv contains prefix like grp_1.. we need to extract address only + hashmap.set( + validator.operator_address, + validator.validator.validator_voting_powers[0].voting_power.toString() + ); + } + } + return hashmap; +} + +async function set_voting_power_for_an_active_validator_in_kv( + validator_address: string, + data: ActiveValidatorsKV +) { + // get validator from kv first + // set it's voting power + const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { + type: 'json', + })) as ActiveValidatorsKV; + if (validator_from_kv) { + validator_from_kv.votingPower = data.votingPower; + const updated_validator_data = JSON.stringify(validator_from_kv); + const validator_group_with_smallest_voting_power = + await get_validator_group_with_smallest_voting_power(); + + const key = `grp_${validator_group_with_smallest_voting_power}:${validator_address}`; + await ACTIVE_VALIDATORS.put(key, updated_validator_data); + console.log('Added new validator to the list', validator_address); + } +} +async function delete_stale_validator_from_kv(key: string) { + await ACTIVE_VALIDATORS.delete(key); + console.log('Deleted stale validator from the list', key); +} + +async function set_total_delegators_count_for_a_validator( + validator_address: string, + total_delegators_count_for_a_validator: number +) { + // get validator from kv first + // set it's total delegators count + const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { + type: 'json', + })) as ActiveValidatorsKV; + if (validator_from_kv) { + validator_from_kv.totalDelegatorsCount = + total_delegators_count_for_a_validator.toString(); + await ACTIVE_VALIDATORS.put( + validator_address, + JSON.stringify(validator_from_kv) + ); + } +} + +async function get_validator_group_with_smallest_voting_power(): Promise { + let voting_power_sum = 0; + let validator_voting_power_total_arr = []; + + for (let i = 1; i <= Number(ACTIVE_VALIDATOR_GROUPS); i++) { + const validator_group = i; + const current_validator_group_data = await ACTIVE_VALIDATORS.list({ + prefix: `grp_${validator_group}:`, + }); + + // accumulates total voting power for specific validator group + for (let validator of current_validator_group_data.keys) { + const validator_data = (await ACTIVE_VALIDATORS.get( + validator.name + )) as ActiveValidatorsKV; + + if (validator_data && validator_data.votingPower) { + voting_power_sum += Number(validator_data.votingPower); + } + } + validator_voting_power_total_arr.push(voting_power_sum); + voting_power_sum = 0; // reset + } + + let smallest_validator_group = 10; // fallback group is 10 + smallest_validator_group = validator_voting_power_total_arr.sort( + (a, b) => a - b + )[0]; // sort by ASC + + return smallest_validator_group; +} +export interface ActiveValidatorsKV { + totalDelegatorsCount?: string; + votingPower?: string; +} From aab5009f681c6c8fd2aadd00364f68e50e47cb0b Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 15:05:39 +0000 Subject: [PATCH 146/224] Remove unused function in node helper --- src/helpers/node.ts | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/src/helpers/node.ts b/src/helpers/node.ts index fcf547dc..c16791a2 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,38 +1,10 @@ import { NodeApi } from '../api/nodeApi'; -import { Account } from '../types/bigDipper'; -import { Coin, DelegationsResponse, UnbondingResponse } from '../types/node'; - -export function total_balance_ncheq(account: Account): number { - let balance = Number( - account?.accountBalance?.coins.find((c) => c.denom === 'ncheq')?.amount || - '0' - ); - - let delegations = 0; - if ( - account?.delegationBalance?.coins && - account?.delegationBalance?.coins.length > 0 - ) { - delegations = Number(account?.delegationBalance?.coins[0].amount); - } - - let unbonding = 0; - if ( - account?.unbondingBalance?.coins && - account?.unbondingBalance?.coins.length > 0 - ) { - unbonding = Number(account?.unbondingBalance?.coins[0]?.amount); - } - - let rewards = 0; - if (account?.rewardBalance?.length > 0) { - for (let i = 0; i < account?.rewardBalance.length; i++) { - rewards += Number(account?.rewardBalance[i]?.coins[0].amount); - } - } - - return balance + delegations + unbonding + rewards; -} +import { + Account, + Coin, + DelegationsResponse, + UnbondingResponse +} from '../types/node'; export function delayed_balance_ncheq(balance: Coin[]): number { return Number(balance.find((c) => c.denom === 'ncheq')?.amount || '0'); From ed2ff3e003dccaabfbf1b65b08e4b2d33d5d4583 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 15:05:48 +0000 Subject: [PATCH 147/224] Remove unused import in vesting --- src/handlers/vestedBalance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index 028eb97c..7cf60b26 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -4,7 +4,7 @@ import { validate_cheqd_address, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; -import { calculate_vested_coins, estimatedVesting } from '../helpers/vesting'; +import { estimatedVesting } from '../helpers/vesting'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; export async function handler(request: Request): Promise { From 1b075556e80262fed2877f83ff0cdca3f5569223 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Wed, 2 Nov 2022 15:05:57 +0000 Subject: [PATCH 148/224] Update delegatorCount.ts --- src/handlers/delegatorCount.ts | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/handlers/delegatorCount.ts b/src/handlers/delegatorCount.ts index a534ad1d..31c7b855 100644 --- a/src/handlers/delegatorCount.ts +++ b/src/handlers/delegatorCount.ts @@ -1,6 +1,5 @@ import { Request } from 'itty-router'; -import { NodeApi } from '../api/nodeApi'; -import { ActiveValidatorsKV } from '../helpers/totalDelegators'; +import { ActiveValidatorsKV } from '../helpers/validators'; export async function handler(request: Request): Promise { const address = request.params?.['validator_address']; @@ -9,22 +8,18 @@ export async function handler(request: Request): Promise { throw new Error('No address specified or wrong address format.'); } - const total_delegators_from_cache = + try { + const total_delegators_from_cache = await try_getting_delegators_count_from_KV(address); - if (total_delegators_from_cache) { - return new Response(total_delegators_from_cache.toString()); + if(!total_delegators_from_cache) { + throw new Error('No delegators count cached for given validator'); + } + return new Response(JSON.stringify(total_delegators_from_cache)); + } + catch (error) { + console.log(error); + throw new Error('Error while getting delegators count for validator'); } - - const node_api = new NodeApi(REST_API); - const resp = await node_api.staking_get_delegators_per_validator( - address, - 0, - true - ); - - const total_delegators = resp.pagination.total; - - return new Response(total_delegators.toString()); } async function try_getting_delegators_count_from_KV(validator_address: string) { From ef0560349c188a95abfd9c7c0bce0d740700cbd8 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:30:40 +0300 Subject: [PATCH 149/224] refactor webhook triggers --- src/handlers/webhookTriggers.ts | 17 ++------- src/helpers/validators.ts | 65 +++++++++++++++++---------------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 78caa611..8f189de4 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,23 +1,14 @@ import { updateCirculatingSupply } from '../helpers/circulating'; -import { updateValidatorKV } from '../helpers/validators'; +import { updateActiveValidatorsKV } from '../helpers/validators'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); - await updateCirculatingSupply(getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS))); - await de - - const gql_client = new GraphQLClient(GRAPHQL_API); - const bd_api = new BigDipperApi(gql_client); - const active_validators_resp = await bd_api.get_active_validators(); - - await remove_any_jailed_validators_from_kv(active_validators_resp); - // also set total delegator count for a validator - await update_delegator_to_validators_KV( - getRandomGroup(Number(ACTIVE_VALIDATOR_GROUPS)) + await updateCirculatingSupply( + getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) ); - await add_new_active_validators_to_kv(active_validators_resp); + await updateActiveValidatorsKV(); } export async function sendPriceDiscrepancies() { diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 68ab327b..d68ead1b 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -2,8 +2,19 @@ import { extract_group_number_and_address } from './circulating'; import { get_all_delegators_for_a_validator } from './node'; import { BigDipperApi } from '../api/bigDipperApi'; import { ActiveValidatorsResponse } from '../types/bigDipper'; +import { GraphQLClient } from './graphql'; +import { NodeApi } from '../api/nodeApi'; -export async function add_new_active_validators_to_kv( +export async function updateActiveValidatorsKV() { + const gql_client = new GraphQLClient(GRAPHQL_API); + const bd_api = new BigDipperApi(gql_client); + const active_validators_resp = await bd_api.get_active_validators(); + + await remove_any_jailed_validators_from_kv(active_validators_resp); + await add_new_active_validators_to_kv(active_validators_resp); +} + +async function add_new_active_validators_to_kv( active_validators: ActiveValidatorsResponse ) { console.log('Adding new active validators to KV, if any...'); @@ -24,18 +35,16 @@ export async function add_new_active_validators_to_kv( if (!is_active_validator_in_kv) { // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. - set_voting_power_for_an_active_validator_in_kv( + put_an_active_validator_in_kv( latest_active_validator.operator_address, - { - votingPower: - latest_active_validator.validator.validator_voting_powers[0].voting_power.toString(), - } + latest_active_validator.validator.validator_voting_powers[0] + .voting_power ); } } } -export async function remove_any_jailed_validators_from_kv( +async function remove_any_jailed_validators_from_kv( active_validators: ActiveValidatorsResponse ) { // list of validators form kv @@ -90,23 +99,35 @@ function create_hashmap_of_validators_addresses_from_api( return hashmap; } -async function set_voting_power_for_an_active_validator_in_kv( +async function put_an_active_validator_in_kv( validator_address: string, - data: ActiveValidatorsKV + voting_power: number ) { // get validator from kv first // set it's voting power const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { type: 'json', })) as ActiveValidatorsKV; + if (validator_from_kv) { - validator_from_kv.votingPower = data.votingPower; - const updated_validator_data = JSON.stringify(validator_from_kv); + const data = {} as ActiveValidatorsKV; + const node_api = new NodeApi(REST_API); + + const delegator_resp = + await node_api.staking_get_all_delegations_for_delegator( + validator_address, + 0, + true + ); + data.totalDelegatorsCount = delegator_resp.pagination.total; + data.updatedAt = new Date().toUTCString(); + data.votingPower = voting_power.toString(); + const validator_group_with_smallest_voting_power = await get_validator_group_with_smallest_voting_power(); const key = `grp_${validator_group_with_smallest_voting_power}:${validator_address}`; - await ACTIVE_VALIDATORS.put(key, updated_validator_data); + await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); console.log('Added new validator to the list', validator_address); } } @@ -115,25 +136,6 @@ async function delete_stale_validator_from_kv(key: string) { console.log('Deleted stale validator from the list', key); } -async function set_total_delegators_count_for_a_validator( - validator_address: string, - total_delegators_count_for_a_validator: number -) { - // get validator from kv first - // set it's total delegators count - const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { - type: 'json', - })) as ActiveValidatorsKV; - if (validator_from_kv) { - validator_from_kv.totalDelegatorsCount = - total_delegators_count_for_a_validator.toString(); - await ACTIVE_VALIDATORS.put( - validator_address, - JSON.stringify(validator_from_kv) - ); - } -} - async function get_validator_group_with_smallest_voting_power(): Promise { let voting_power_sum = 0; let validator_voting_power_total_arr = []; @@ -168,4 +170,5 @@ async function get_validator_group_with_smallest_voting_power(): Promise export interface ActiveValidatorsKV { totalDelegatorsCount?: string; votingPower?: string; + updatedAt?: string; } From f72291c46b58f356fd162aa0827665615c6c8a7f Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:08:35 +0300 Subject: [PATCH 150/224] refactor circulating --- src/handlers/circulatingSupply.ts | 40 +------------------------------ src/helpers/circulating.ts | 38 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 3289e858..3fc71810 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,43 +1,5 @@ import { Request } from 'itty-router'; -import { ncheq_to_cheq_fixed } from '../helpers/currency'; -import { NodeApi } from '../api/nodeApi'; -import { AccountBalanceInfos } from '../types/node'; - -async function get_total_supply(): Promise { - let node_api = new NodeApi(REST_API); - let total_supply_ncheq = await node_api.bank_get_total_supply_ncheq(); - const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); - - return total_supply; -} - -async function get_circulating_supply(): Promise { - const total_supply = await get_total_supply(); - - try { - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); - console.log(`Total cached entries: ${cached.keys.length}`); - - let shareholders_total_balance = Number(0); - for (const key of cached.keys) { - let data: AccountBalanceInfos | null = - await CIRCULATING_SUPPLY_WATCHLIST.get(key.name, { - type: 'json', - }); - - if (data !== null && data.totalBalance !== null) { - shareholders_total_balance += Number(data.totalBalance); - } - } - - console.log('Total supply', total_supply); - console.log(`Watchlist total balance: ${shareholders_total_balance}`); - - return total_supply - shareholders_total_balance; - } catch (e: any) { - throw new Error(e.toString); - } -} +import { get_circulating_supply } from '../helpers/circulating'; export async function handler(request: Request): Promise { let circulating_supply = await get_circulating_supply(); diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 1ed4f4ae..487591d6 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -1,4 +1,7 @@ import { get_account_balance_infos_from_node_api } from './balance'; +import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { NodeApi } from '../api/nodeApi'; +import { AccountBalanceInfos } from '../types/node'; export function extract_group_number_and_address(key: string) { const parts = key.split(':'); @@ -56,3 +59,38 @@ export async function updateCachedBalance(addr: string, grpN: number) { console.log(`error updateCachedBalance: ${e}`); } } + +export async function get_total_supply(): Promise { + let node_api = new NodeApi(REST_API); + let total_supply_ncheq = await node_api.bank_get_total_supply_ncheq(); + const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); + + return total_supply; +} + +export async function get_circulating_supply(): Promise { + const total_supply = await get_total_supply(); + + try { + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); + console.log(`Total cached entries: ${cached.keys.length}`); + let shareholders_total_balance = Number(0); + for (const key of cached.keys) { + let data: AccountBalanceInfos | null = + await CIRCULATING_SUPPLY_WATCHLIST.get(key.name, { + type: 'json', + }); + + if (data !== null && data.totalBalance !== null) { + shareholders_total_balance += Number(data.totalBalance); + } + } + + console.log('Total supply', total_supply); + console.log(`Watchlist total balance: ${shareholders_total_balance}`); + + return total_supply - shareholders_total_balance; + } catch (e: any) { + throw new Error(e.toString); + } +} From 41a5859835d4dfe371f121fec1d6b21e7ecdfdb7 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:14:29 +0300 Subject: [PATCH 151/224] refactor delegators count --- src/handlers/delegatorCount.ts | 19 ++++--------------- src/helpers/validators.ts | 17 ++++++++++++----- src/types/kv.ts | 5 +++++ 3 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 src/types/kv.ts diff --git a/src/handlers/delegatorCount.ts b/src/handlers/delegatorCount.ts index 31c7b855..4715df6e 100644 --- a/src/handlers/delegatorCount.ts +++ b/src/handlers/delegatorCount.ts @@ -1,5 +1,5 @@ import { Request } from 'itty-router'; -import { ActiveValidatorsKV } from '../helpers/validators'; +import { try_getting_delegators_count_from_KV } from '../helpers/validators'; export async function handler(request: Request): Promise { const address = request.params?.['validator_address']; @@ -10,24 +10,13 @@ export async function handler(request: Request): Promise { try { const total_delegators_from_cache = - await try_getting_delegators_count_from_KV(address); - if(!total_delegators_from_cache) { + await try_getting_delegators_count_from_KV(address); + if (!total_delegators_from_cache) { throw new Error('No delegators count cached for given validator'); } return new Response(JSON.stringify(total_delegators_from_cache)); - } - catch (error) { + } catch (error) { console.log(error); throw new Error('Error while getting delegators count for validator'); } } - -async function try_getting_delegators_count_from_KV(validator_address: string) { - const validator_data = ACTIVE_VALIDATORS.get( - validator_address - ) as ActiveValidatorsKV; - - return validator_data.totalDelegatorsCount - ? validator_data.totalDelegatorsCount - : null; -} diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index d68ead1b..794e16be 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -1,9 +1,9 @@ import { extract_group_number_and_address } from './circulating'; -import { get_all_delegators_for_a_validator } from './node'; import { BigDipperApi } from '../api/bigDipperApi'; import { ActiveValidatorsResponse } from '../types/bigDipper'; import { GraphQLClient } from './graphql'; import { NodeApi } from '../api/nodeApi'; +import { ActiveValidatorsKV } from '../types/KV'; export async function updateActiveValidatorsKV() { const gql_client = new GraphQLClient(GRAPHQL_API); @@ -167,8 +167,15 @@ async function get_validator_group_with_smallest_voting_power(): Promise return smallest_validator_group; } -export interface ActiveValidatorsKV { - totalDelegatorsCount?: string; - votingPower?: string; - updatedAt?: string; + +export async function try_getting_delegators_count_from_KV( + validator_address: string +) { + const validator_data = ACTIVE_VALIDATORS.get( + validator_address + ) as ActiveValidatorsKV; + + return validator_data.totalDelegatorsCount + ? validator_data.totalDelegatorsCount + : null; } diff --git a/src/types/kv.ts b/src/types/kv.ts new file mode 100644 index 00000000..71652abd --- /dev/null +++ b/src/types/kv.ts @@ -0,0 +1,5 @@ +export interface ActiveValidatorsKV { + totalDelegatorsCount?: string; + votingPower?: string; + updatedAt?: string; +} From 964dff119c06afc639950386e0a300ca431ca7e8 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:19:58 +0300 Subject: [PATCH 152/224] cleanup vesting --- src/handlers/vestingBalance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index 10d6ff3f..77c8bcf3 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -4,7 +4,7 @@ import { validate_cheqd_address, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; -import { calculate_vesting_coins, estimatedVesting } from '../helpers/vesting'; +import { estimatedVesting } from '../helpers/vesting'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; export async function handler(request: Request): Promise { From 0b2d74ec2ca7dbb63e0fd041116d660c5660e013 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:25:06 +0300 Subject: [PATCH 153/224] refactor helpers: balance --- src/helpers/balance.ts | 79 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index e9d30d33..ad8d9572 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,10 +1,7 @@ import { NodeApi } from '../api/nodeApi'; import { AccountBalanceInfos } from '../types/node'; import { ncheq_to_cheq_fixed } from './currency'; -import { - calculate_total_delegations_balance_for_delegator_in_ncheq, - calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq, -} from './node'; +import { DelegationsResponse, UnbondingResponse } from '../types/node'; export async function get_account_balance_infos_from_node_api( address: string @@ -56,3 +53,77 @@ export async function get_account_balance_infos_from_node_api( timeUpdated: new Date().toUTCString(), }; } + +export async function calculate_total_delegations_balance_for_delegator_in_ncheq( + delegationsResp: DelegationsResponse, + current_offset: number +): Promise { + let total_delegation_balance_in_ncheq = 0; + const total_count = Number(delegationsResp.pagination.total); + + for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { + total_delegation_balance_in_ncheq += Number( + delegationsResp.delegation_responses[i].balance.amount + ); + } + + if (current_offset < total_count) { + const node_api = new NodeApi(REST_API); + const delegator_address = + delegationsResp.delegation_responses[0].delegation.delegator_address; + + const resp = await node_api.staking_get_all_delegations_for_delegator( + delegator_address, + current_offset, // our current offset will be updated by recursive call below + true // we count total again , since it's implemented recursively + ); + + total_delegation_balance_in_ncheq += + await calculate_total_delegations_balance_for_delegator_in_ncheq( + resp, + current_offset + Number(REST_API_PAGINATION_LIMIT) + ); + } + + return total_delegation_balance_in_ncheq; +} + +export async function calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( + unbondingResp: UnbondingResponse, + current_offset: number +): Promise { + let total_unbonding_balance_in_ncheq = 0; + const total_count = Number(unbondingResp.pagination.total); + for (let i = 0; i < unbondingResp.unbonding_responses.length; i++) { + for ( + let j = 0; + j < unbondingResp.unbonding_responses[i].entries.length; + j++ + ) { + total_unbonding_balance_in_ncheq += Number( + unbondingResp.unbonding_responses[i].entries[j].balance + ); + } + } + + if (current_offset < total_count) { + const node_api = new NodeApi(REST_API); + const delegator_address = + unbondingResp.unbonding_responses[0].delegator_address; + + const resp = + await node_api.staking_get_all_unbonding_delegations_for_delegator( + delegator_address, + current_offset, + true + ); + + total_unbonding_balance_in_ncheq += + await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( + resp, + current_offset + Number(REST_API_PAGINATION_LIMIT) + ); + } + + return total_unbonding_balance_in_ncheq; +} From 1cda6630604bd8af49d5b5cffbfc27087e429f2b Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:25:27 +0300 Subject: [PATCH 154/224] refactor helpers node --- src/helpers/node.ts | 81 +-------------------------------------------- 1 file changed, 1 insertion(+), 80 deletions(-) diff --git a/src/helpers/node.ts b/src/helpers/node.ts index c16791a2..a7cf084f 100644 --- a/src/helpers/node.ts +++ b/src/helpers/node.ts @@ -1,89 +1,10 @@ import { NodeApi } from '../api/nodeApi'; -import { - Account, - Coin, - DelegationsResponse, - UnbondingResponse -} from '../types/node'; +import { Coin } from '../types/node'; export function delayed_balance_ncheq(balance: Coin[]): number { return Number(balance.find((c) => c.denom === 'ncheq')?.amount || '0'); } -export async function calculate_total_delegations_balance_for_delegator_in_ncheq( - delegationsResp: DelegationsResponse, - current_offset: number -): Promise { - let total_delegation_balance_in_ncheq = 0; - const total_count = Number(delegationsResp.pagination.total); - - for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { - total_delegation_balance_in_ncheq += Number( - delegationsResp.delegation_responses[i].balance.amount - ); - } - - if (current_offset < total_count) { - const node_api = new NodeApi(REST_API); - const delegator_address = - delegationsResp.delegation_responses[0].delegation.delegator_address; - - const resp = await node_api.staking_get_all_delegations_for_delegator( - delegator_address, - current_offset, // our current offset will be updated by recursive call below - true // we count total again , since it's implemented recursively - ); - - total_delegation_balance_in_ncheq += - await calculate_total_delegations_balance_for_delegator_in_ncheq( - resp, - current_offset + Number(REST_API_PAGINATION_LIMIT) - ); - } - - return total_delegation_balance_in_ncheq; -} - -export async function calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( - unbondingResp: UnbondingResponse, - current_offset: number -): Promise { - let total_unbonding_balance_in_ncheq = 0; - const total_count = Number(unbondingResp.pagination.total); - for (let i = 0; i < unbondingResp.unbonding_responses.length; i++) { - for ( - let j = 0; - j < unbondingResp.unbonding_responses[i].entries.length; - j++ - ) { - total_unbonding_balance_in_ncheq += Number( - unbondingResp.unbonding_responses[i].entries[j].balance - ); - } - } - - if (current_offset < total_count) { - const node_api = new NodeApi(REST_API); - const delegator_address = - unbondingResp.unbonding_responses[0].delegator_address; - - const resp = - await node_api.staking_get_all_unbonding_delegations_for_delegator( - delegator_address, - current_offset, - true - ); - - total_unbonding_balance_in_ncheq += - await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( - resp, - current_offset + Number(REST_API_PAGINATION_LIMIT) - ); - } - - return total_unbonding_balance_in_ncheq; -} - export async function get_all_delegators_for_a_validator( validator_address: string ): Promise { From 322d4e76e8d369f75c039cc466b57aa0f1b26299 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:35:06 +0300 Subject: [PATCH 155/224] move group and address extraction to separate file --- src/helpers/circulating.ts | 11 +---------- src/helpers/kv.ts | 9 +++++++++ src/helpers/validators.ts | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 src/helpers/kv.ts diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 487591d6..25b6dd88 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -2,16 +2,7 @@ import { get_account_balance_infos_from_node_api } from './balance'; import { ncheq_to_cheq_fixed } from '../helpers/currency'; import { NodeApi } from '../api/nodeApi'; import { AccountBalanceInfos } from '../types/node'; - -export function extract_group_number_and_address(key: string) { - const parts = key.split(':'); - let addr = parts[1]; - let grpN = Number(parts[0].split('_')[1]); - return { - address: addr, - groupNumber: grpN, - }; -} +import { extract_group_number_and_address } from './kv'; export async function updateCirculatingSupply(groupNumber: number) { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ diff --git a/src/helpers/kv.ts b/src/helpers/kv.ts new file mode 100644 index 00000000..bedde805 --- /dev/null +++ b/src/helpers/kv.ts @@ -0,0 +1,9 @@ +export function extract_group_number_and_address(key: string) { + const parts = key.split(':'); + let addr = parts[1]; + let grpN = Number(parts[0].split('_')[1]); + return { + address: addr, + groupNumber: grpN, + }; +} diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 794e16be..7f56ea88 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -1,4 +1,4 @@ -import { extract_group_number_and_address } from './circulating'; +import { extract_group_number_and_address } from './kv'; import { BigDipperApi } from '../api/bigDipperApi'; import { ActiveValidatorsResponse } from '../types/bigDipper'; import { GraphQLClient } from './graphql'; From 24efdd099d21a1930f576ae144addcdbf384e487 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:40:38 +0300 Subject: [PATCH 156/224] remove old kv as we dont need pre-populated kv --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 0f57f3a0..6c312db5 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -103,7 +103,7 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "232b03c1c3784d17b242d4afcd07dec0" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "d78a08202b1941bcb24413d3a2891d12" } ] # Cron triggers for staging worker From 9ad77312dc376874555d139cc0d88853f8af81e7 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 12:21:49 +0300 Subject: [PATCH 157/224] add limit optional param --- src/api/nodeApi.ts | 9 +++++--- src/helpers/validators.ts | 44 ++++++--------------------------------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index bc29de3d..e3148585 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -40,7 +40,7 @@ export class NodeApi { let resp = await fetch( `${this.base_rest_api_url}/cosmos/distribution/v1beta1/delegators/${address}/rewards` ); - let respJson = (await resp.json()) as RewardsResponse + let respJson = (await resp.json()) as RewardsResponse; return Number(respJson?.total?.[0]?.amount ?? '0'); } @@ -67,13 +67,16 @@ export class NodeApi { async staking_get_all_delegations_for_delegator( address: string, offset: number, - should_count_total: boolean + should_count_total: boolean, + limit?: number ) { // order of query params: count_total -> offset -> limit const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_limit = `pagination.limit=${ + limit ? limit : REST_API_PAGINATION_LIMIT + }`; const pagination_offset = `pagination.offset=${offset}`; // NOTE: be cautious of newlines or spaces. Might make the request URL malformed const resp = await fetch( diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 7f56ea88..01c6a297 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -109,7 +109,8 @@ async function put_an_active_validator_in_kv( type: 'json', })) as ActiveValidatorsKV; - if (validator_from_kv) { + if (!validator_from_kv) { + console.log('putting new active validator', validator_address); const data = {} as ActiveValidatorsKV; const node_api = new NodeApi(REST_API); @@ -117,16 +118,15 @@ async function put_an_active_validator_in_kv( await node_api.staking_get_all_delegations_for_delegator( validator_address, 0, - true + true, + 1 ); + data.totalDelegatorsCount = delegator_resp.pagination.total; data.updatedAt = new Date().toUTCString(); data.votingPower = voting_power.toString(); - const validator_group_with_smallest_voting_power = - await get_validator_group_with_smallest_voting_power(); - - const key = `grp_${validator_group_with_smallest_voting_power}:${validator_address}`; + const key = `${validator_address}`; await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); console.log('Added new validator to the list', validator_address); } @@ -136,38 +136,6 @@ async function delete_stale_validator_from_kv(key: string) { console.log('Deleted stale validator from the list', key); } -async function get_validator_group_with_smallest_voting_power(): Promise { - let voting_power_sum = 0; - let validator_voting_power_total_arr = []; - - for (let i = 1; i <= Number(ACTIVE_VALIDATOR_GROUPS); i++) { - const validator_group = i; - const current_validator_group_data = await ACTIVE_VALIDATORS.list({ - prefix: `grp_${validator_group}:`, - }); - - // accumulates total voting power for specific validator group - for (let validator of current_validator_group_data.keys) { - const validator_data = (await ACTIVE_VALIDATORS.get( - validator.name - )) as ActiveValidatorsKV; - - if (validator_data && validator_data.votingPower) { - voting_power_sum += Number(validator_data.votingPower); - } - } - validator_voting_power_total_arr.push(voting_power_sum); - voting_power_sum = 0; // reset - } - - let smallest_validator_group = 10; // fallback group is 10 - smallest_validator_group = validator_voting_power_total_arr.sort( - (a, b) => a - b - )[0]; // sort by ASC - - return smallest_validator_group; -} - export async function try_getting_delegators_count_from_KV( validator_address: string ) { From c669e3b4eccba93530cf42c5d07bd01fc222b5de Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 12:27:51 +0300 Subject: [PATCH 158/224] fix adding new validator to kv logic --- src/helpers/validators.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 01c6a297..1527e3e6 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -67,11 +67,14 @@ async function remove_any_jailed_validators_from_kv( } function create_hashmap_of_validators_addresses_from_kv( - validators_from_KV: KVNamespaceListKey[] + validators_from_KV?: KVNamespaceListKey[] ): Map { + const hashmap = new Map(); // keys incase contain prefixes, since they are from KV + if (!validators_from_KV) { + return hashmap; + } - const hashmap = new Map(); for (let key of validators_from_KV) { const key_to_look_up = extract_group_number_and_address(key.name).address; if (!hashmap.has(key_to_look_up)) { @@ -119,7 +122,7 @@ async function put_an_active_validator_in_kv( validator_address, 0, true, - 1 + 1 // set limit param to 1, lessen stress on node api ); data.totalDelegatorsCount = delegator_resp.pagination.total; From f5c7820cc1e678d559cb245c6beb786a7fc56368 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 12:32:46 +0300 Subject: [PATCH 159/224] fix creating map logic --- src/helpers/validators.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 1527e3e6..739b38f5 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -57,10 +57,7 @@ async function remove_any_jailed_validators_from_kv( create_hashmap_of_validators_addresses_from_api(active_validators); for (let validator_from_kv of active_validators_from_kv.keys) { - const key_to_look_up = extract_group_number_and_address( - validator_from_kv.name - ).address; - if (!active_validators_from_api_hash_map.has(key_to_look_up)) { + if (!active_validators_from_api_hash_map.has(validator_from_kv.name)) { delete_stale_validator_from_kv(validator_from_kv.name); } } @@ -76,10 +73,9 @@ function create_hashmap_of_validators_addresses_from_kv( } for (let key of validators_from_KV) { - const key_to_look_up = extract_group_number_and_address(key.name).address; - if (!hashmap.has(key_to_look_up)) { + if (!hashmap.has(key.name)) { // since kv contains prefix like grp_1.. we need to extract address only - hashmap.set(key_to_look_up, key_to_look_up); + hashmap.set(key.name, key.name); } } return hashmap; From 3fda52be57f790bb0ea020273df2f5206927fd60 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 12:57:27 +0300 Subject: [PATCH 160/224] remove sending arbitrage as it is causing hook to fail --- src/handlers/webhookTriggers.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 8f189de4..3b5bdeaf 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -4,7 +4,8 @@ import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); - await sendPriceDiscrepancies(); + // await sendPriceDiscrepancies(); // removed to test + await updateCirculatingSupply( getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) ); From 654f33d2a1cb9b2f43bc7c0ad3a12b70addcef7f Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 13:40:31 +0300 Subject: [PATCH 161/224] remove additional check when adding validator to KV --- src/helpers/validators.ts | 42 ++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 739b38f5..7f884bed 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -102,33 +102,25 @@ async function put_an_active_validator_in_kv( validator_address: string, voting_power: number ) { - // get validator from kv first - // set it's voting power - const validator_from_kv = (await ACTIVE_VALIDATORS.get(validator_address, { - type: 'json', - })) as ActiveValidatorsKV; - - if (!validator_from_kv) { - console.log('putting new active validator', validator_address); - const data = {} as ActiveValidatorsKV; - const node_api = new NodeApi(REST_API); - - const delegator_resp = - await node_api.staking_get_all_delegations_for_delegator( - validator_address, - 0, - true, - 1 // set limit param to 1, lessen stress on node api - ); + console.log('putting new active validator', validator_address); + const data = {} as ActiveValidatorsKV; + const node_api = new NodeApi(REST_API); + + const delegator_resp = + await node_api.staking_get_all_delegations_for_delegator( + validator_address, + 0, + true, + 1 // set limit param to 1, lessen stress on node api + ); - data.totalDelegatorsCount = delegator_resp.pagination.total; - data.updatedAt = new Date().toUTCString(); - data.votingPower = voting_power.toString(); + data.totalDelegatorsCount = delegator_resp.pagination.total; + data.updatedAt = new Date().toUTCString(); + data.votingPower = voting_power.toString(); - const key = `${validator_address}`; - await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); - console.log('Added new validator to the list', validator_address); - } + const key = `${validator_address}`; + await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); + console.log('Added new validator to the list', validator_address); } async function delete_stale_validator_from_kv(key: string) { await ACTIVE_VALIDATORS.delete(key); From 0104e554cec5a7002de50a81cca9ce1db6a008ed Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 13:57:22 +0300 Subject: [PATCH 162/224] add additional logs --- src/api/nodeApi.ts | 2 ++ src/helpers/validators.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index e3148585..071f0749 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -83,6 +83,8 @@ export class NodeApi { `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); + console.log(`Response status for delegator count ${resp.status}`); + return (await resp.json()) as DelegationsResponse; } diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 7f884bed..bd892d30 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -4,6 +4,7 @@ import { ActiveValidatorsResponse } from '../types/bigDipper'; import { GraphQLClient } from './graphql'; import { NodeApi } from '../api/nodeApi'; import { ActiveValidatorsKV } from '../types/KV'; +import { resolve } from 'path'; export async function updateActiveValidatorsKV() { const gql_client = new GraphQLClient(GRAPHQL_API); @@ -118,6 +119,8 @@ async function put_an_active_validator_in_kv( data.updatedAt = new Date().toUTCString(); data.votingPower = voting_power.toString(); + console.log(`Validator data ${data}`); + const key = `${validator_address}`; await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); console.log('Added new validator to the list', validator_address); From dcfa31ee9cdef290479e5466ea9b326176005e20 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 13:59:20 +0300 Subject: [PATCH 163/224] remove old pre populated validators kv for prod --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 6c312db5..ffbb99f0 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -33,7 +33,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" }, { binding = "ACTIVE_VALIDATORS", id = "02a25d1f74f44e22ab3ee7c290c4b8e0" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" }, { binding = "ACTIVE_VALIDATORS", id = "b9afd59ab39b4609905775bd651efd7f" } ] # Map of environment variables to set when deploying the Worker From 75a17436aa1770a5b62ff953403f55069551e0ae Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 12:50:25 +0000 Subject: [PATCH 164/224] Fix imports --- src/helpers/validators.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index bd892d30..7ce51374 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -1,10 +1,8 @@ -import { extract_group_number_and_address } from './kv'; import { BigDipperApi } from '../api/bigDipperApi'; import { ActiveValidatorsResponse } from '../types/bigDipper'; import { GraphQLClient } from './graphql'; import { NodeApi } from '../api/nodeApi'; -import { ActiveValidatorsKV } from '../types/KV'; -import { resolve } from 'path'; +import { ActiveValidatorsKV } from '../types/kv'; export async function updateActiveValidatorsKV() { const gql_client = new GraphQLClient(GRAPHQL_API); @@ -48,10 +46,8 @@ async function add_new_active_validators_to_kv( async function remove_any_jailed_validators_from_kv( active_validators: ActiveValidatorsResponse ) { - // list of validators form kv - // list of validators from api - // loop throu validators from kv, and if - // a validator from kv doesnt exist in api remove the kv + // Loop through validators from kv, and if + // a validator from KV doesn't exist in API remove from KV console.log('Removing jailed validators, if any'); const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_api_hash_map = @@ -68,7 +64,7 @@ function create_hashmap_of_validators_addresses_from_kv( validators_from_KV?: KVNamespaceListKey[] ): Map { const hashmap = new Map(); - // keys incase contain prefixes, since they are from KV + // In case keys contain prefixes, since they are from KV if (!validators_from_KV) { return hashmap; } @@ -85,7 +81,7 @@ function create_hashmap_of_validators_addresses_from_kv( function create_hashmap_of_validators_addresses_from_api( validators_from_api: ActiveValidatorsResponse ): Map { - // keys incase contain prefixes, since they are from KV + // In case keys contain prefixes, since they are from KV const hashmap = new Map(); for (let validator of validators_from_api.validator_info) { if (!hashmap.has(validator.operator_address)) { @@ -133,7 +129,7 @@ async function delete_stale_validator_from_kv(key: string) { export async function try_getting_delegators_count_from_KV( validator_address: string ) { - const validator_data = ACTIVE_VALIDATORS.get( + const validator_data = await ACTIVE_VALIDATORS.get( validator_address ) as ActiveValidatorsKV; From 4484d75dd4e13da1f8458b4622e1ff6cf6cf4b34 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 13:00:06 +0000 Subject: [PATCH 165/224] Remove unncessary helper --- src/api/nodeApi.ts | 8 ++++++-- src/helpers/node.ts | 43 --------------------------------------- src/helpers/validators.ts | 2 +- 3 files changed, 7 insertions(+), 46 deletions(-) delete mode 100644 src/helpers/node.ts diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 071f0749..ab312cf9 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -3,6 +3,7 @@ import { Coin, DelegationsResponse, UnbondingResponse, + RewardsResponse, ValidatorDetailResponse, } from '../types/node'; @@ -48,13 +49,16 @@ export class NodeApi { async staking_get_delegators_per_validator( address: string, offset: number, - should_count_total: boolean + should_count_total: boolean, + limit?: number ): Promise { // order of query params: count_total -> offset -> limit const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `pagination.limit=${REST_API_PAGINATION_LIMIT}`; + const pagination_limit = `pagination.limit=${ + limit ? limit : REST_API_PAGINATION_LIMIT + }`; const pagination_offset = `pagination.offset=${offset}`; // NOTE: be cautious of newlines or spaces. Might make the request URL malformed let resp = await fetch( diff --git a/src/helpers/node.ts b/src/helpers/node.ts deleted file mode 100644 index a7cf084f..00000000 --- a/src/helpers/node.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { NodeApi } from '../api/nodeApi'; -import { Coin } from '../types/node'; - -export function delayed_balance_ncheq(balance: Coin[]): number { - return Number(balance.find((c) => c.denom === 'ncheq')?.amount || '0'); -} - -export async function get_all_delegators_for_a_validator( - validator_address: string -): Promise { - const node_api = new NodeApi(REST_API); - let offset = 0; - - let delegationsResp = await node_api.staking_get_delegators_per_validator( - validator_address, - offset, - true // we set it to true to get total_delegators_count - ); - const total_delegators_count = Number(delegationsResp.pagination.total); - const delegators = []; - - while ( - offset < total_delegators_count || - delegationsResp.delegation_responses.length > 0 - ) { - for (let i = 0; i < delegationsResp.delegation_responses.length; i++) { - const delegator = - delegationsResp.delegation_responses[i].delegation.delegator_address; - delegators.push(delegator); - } - offset += Number(REST_API_PAGINATION_LIMIT); - delegationsResp = await node_api.staking_get_delegators_per_validator( - validator_address, - offset, - false // we dont need to get total_count on subsequent queries - ); - if (offset > total_delegators_count) { - break; - } - } - - return delegators; -} diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 7ce51374..6ee3a2f1 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -104,7 +104,7 @@ async function put_an_active_validator_in_kv( const node_api = new NodeApi(REST_API); const delegator_resp = - await node_api.staking_get_all_delegations_for_delegator( + await node_api.staking_get_delegators_per_validator( validator_address, 0, true, From c1202b757d44452391482e5c3c0b587793572076 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 13:04:47 +0000 Subject: [PATCH 166/224] Update balance.ts --- src/helpers/balance.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index ad8d9572..079c7660 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -1,7 +1,11 @@ import { NodeApi } from '../api/nodeApi'; -import { AccountBalanceInfos } from '../types/node'; +import { + AccountBalanceInfos, + DelegationsResponse, + UnbondingResponse +} from '../types/node'; import { ncheq_to_cheq_fixed } from './currency'; -import { DelegationsResponse, UnbondingResponse } from '../types/node'; +import { } from '../types/node'; export async function get_account_balance_infos_from_node_api( address: string From ef9b9b44480c8337cdb23acd29429dc337337ec1 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 13:06:03 +0000 Subject: [PATCH 167/224] Remove total delegators endpoint from README --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index 4558cdaf..203e02d7 100644 --- a/README.md +++ b/README.md @@ -65,20 +65,6 @@ Overall tokens staked, in CHEQ. Provides the overall amount staked pulled from the block explorer. -### ➕ Overall number of delegators - -#### Endpoint - -[`data-api.cheqd.io/staking/delegators/total`](https://data-api.cheqd.io/staking/delegators/total) - -#### Response - -Total number of delegators across every validator on the network. - -#### Rationale - -The only way to derive this figure from the Cosmos SDK APIs is by iterating over every validator and counting the number of delegators. - ### đŸ—ŗ Delegator count by validator #### Endpoint From a9abc4634ffdca3b0fe17f31240686f6f2680860 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 13:06:14 +0000 Subject: [PATCH 168/224] Bump wrangler --- package-lock.json | 26 +++++++++++++------------- package.json | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index c29a674f..fa6c6ac6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "@cloudflare/workers-types": "^3.18.0", "@types/node": "^17.0.45", "typescript": "^4.8.4", - "wrangler": "^2.1.13" + "wrangler": "^2.1.15" } }, "node_modules/@cloudflare/kv-asset-handler": { @@ -1481,9 +1481,9 @@ } }, "node_modules/wrangler": { - "version": "2.1.13", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.13.tgz", - "integrity": "sha512-FWarJ9pBaXOU/wj3BoLo1Azi4VvadD0PfDIYfvY9hoKVyPMSr4dpPNUGgtMhsVuDp7K9mdixnmGEJxR7pbs3kQ==", + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.15.tgz", + "integrity": "sha512-5iqtFNo+zbu1FTnQQU/1Y+WWxIEuPIy71fe0uvqqFl0pSlkAtZJ+ufw8UYVxf2Mprw4ia4mSDdhV+hHpZO1sLQ==", "dev": true, "dependencies": { "@cloudflare/kv-asset-handler": "^0.2.0", @@ -1535,9 +1535,9 @@ } }, "node_modules/xxhash-wasm": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.0.1.tgz", - "integrity": "sha512-Lc9CTvDrH2vRoiaUzz25q7lRaviMhz90pkx6YxR9EPYtF99yOJnv2cB+CQ0hp/TLoqrUsk8z/W2EN31T568Azw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.0.2.tgz", + "integrity": "sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A==", "dev": true }, "node_modules/yallist": { @@ -2553,9 +2553,9 @@ } }, "wrangler": { - "version": "2.1.13", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.13.tgz", - "integrity": "sha512-FWarJ9pBaXOU/wj3BoLo1Azi4VvadD0PfDIYfvY9hoKVyPMSr4dpPNUGgtMhsVuDp7K9mdixnmGEJxR7pbs3kQ==", + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-2.1.15.tgz", + "integrity": "sha512-5iqtFNo+zbu1FTnQQU/1Y+WWxIEuPIy71fe0uvqqFl0pSlkAtZJ+ufw8UYVxf2Mprw4ia4mSDdhV+hHpZO1sLQ==", "dev": true, "requires": { "@cloudflare/kv-asset-handler": "^0.2.0", @@ -2584,9 +2584,9 @@ "requires": {} }, "xxhash-wasm": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.0.1.tgz", - "integrity": "sha512-Lc9CTvDrH2vRoiaUzz25q7lRaviMhz90pkx6YxR9EPYtF99yOJnv2cB+CQ0hp/TLoqrUsk8z/W2EN31T568Azw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.0.2.tgz", + "integrity": "sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A==", "dev": true }, "yallist": { diff --git a/package.json b/package.json index 28e5f6e9..d363610c 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "@cloudflare/workers-types": "^3.18.0", "@types/node": "^17.0.45", "typescript": "^4.8.4", - "wrangler": "^2.1.13" + "wrangler": "^2.1.15" }, "private": true } From 6fb3cbe050195b0669f659066330730c29778534 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 13:08:16 +0000 Subject: [PATCH 169/224] Add cache cleanup scripts --- .github/workflows/cleanup-actions.yml | 45 +++++++++++++++++++++++++++ .github/workflows/cleanup-cache.yml | 21 +++++++++++++ .github/workflows/deploy.yml | 14 +++++++++ 3 files changed, 80 insertions(+) create mode 100644 .github/workflows/cleanup-actions.yml create mode 100644 .github/workflows/cleanup-cache.yml diff --git a/.github/workflows/cleanup-actions.yml b/.github/workflows/cleanup-actions.yml new file mode 100644 index 00000000..ad82df31 --- /dev/null +++ b/.github/workflows/cleanup-actions.yml @@ -0,0 +1,45 @@ +name: "Cleanup - Actions" +on: + workflow_dispatch: + inputs: + days: + description: 'Retain days' + required: true + type: string + default: 30 + minimum_runs: + description: 'Minimum runs to keep for each workflow' + required: true + type: string + default: 0 + delete_workflow_pattern: + description: 'Name/filename of workflow. Default is all.' + required: false + type: string + delete_workflow_by_state_pattern: + description: 'Remove workflow by state: active, deleted, disabled_fork, disabled_inactivity, disabled_manually' + required: true + default: All + type: choice + options: + - All + - active + - deleted + - disabled_inactivity + - disabled_manually + +jobs: + + delete-runs: + name: "Delete old workflow runs" + runs-on: ubuntu-latest + + steps: + - uses: Mattraks/delete-workflow-runs@v2 + with: + token: ${{ github.token }} + repository: ${{ github.repository }} + retain_days: ${{ github.event.inputs.days }} + keep_minimum_runs: ${{ github.event.inputs.minimum_runs }} + delete_workflow_pattern: ${{ github.event.inputs.delete_workflow_pattern }} + delete_workflow_by_state_pattern: ${{ github.event.inputs.delete_workflow_by_state_pattern }} diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml new file mode 100644 index 00000000..09697c19 --- /dev/null +++ b/.github/workflows/cleanup-cache.yml @@ -0,0 +1,21 @@ +name: "Cleanup - Cache" +on: + workflow_dispatch: + inputs: + dry-run: + description: "Dry run only?" + required: true + type: boolean + default: false + +jobs: + + delete-caches: + name: "Delete Actions caches" + runs-on: ubuntu-latest + + steps: + - name: "Wipe Github Actions cache" + uses: easimon/wipe-cache@v1 + with: + dry-run: ${{ github.event.inputs.dry-run }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 05fc6c29..503af3ab 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -70,3 +70,17 @@ jobs: WEBHOOK_URL env: WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} + + cache-purge: + name: "Purge branch Actions cache" + needs: staging-deploy + runs-on: ubuntu-latest + + steps: + + - name: "Delete Branch Cache Action" + uses: snnaplab/delete-branch-cache-action@v1.0.0 + with: + # Specify explicitly because the ref at the time of merging will be a branch name such as 'main', 'develop' + ref: refs/pull/${{ github.event.number }}/merge + github-token: ${{ secrets.GITHUB_TOKEN }} From 797abad939068155dc1ecc782b68ff4215694ae1 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 16:22:19 +0300 Subject: [PATCH 170/224] catch all cron exceptions --- src/handlers/webhookTriggers.ts | 42 ++++++++++++++----------- src/helpers/circulating.ts | 56 ++++++++++++++++++--------------- src/helpers/validators.ts | 33 ++++++++++--------- 3 files changed, 72 insertions(+), 59 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 3b5bdeaf..20e7117c 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -4,7 +4,7 @@ import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); - // await sendPriceDiscrepancies(); // removed to test + await sendPriceDiscrepancies(); // removed to test await updateCirculatingSupply( getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) @@ -13,27 +13,31 @@ export async function webhookTriggers(event: Event) { } export async function sendPriceDiscrepancies() { - console.log('Sending price discrepancies...'); + try { + console.log('Sending price discrepancies...'); - const arbitrageOpportunities = await filterArbitrageOpportunities(); - const hasArbitrageOpportunities = arbitrageOpportunities.length > 0; - if (hasArbitrageOpportunities) { - console.log('Arbitrage opportunities...'); - try { - const init = { - body: JSON.stringify({ - arbitrage_opportunities: arbitrageOpportunities, - }), - method: 'POST', - headers: { - 'content-type': 'application/json;charset=UTF-8', - }, - }; + const arbitrageOpportunities = await filterArbitrageOpportunities(); + const hasArbitrageOpportunities = arbitrageOpportunities.length > 0; + if (hasArbitrageOpportunities) { + console.log('Arbitrage opportunities...'); + try { + const init = { + body: JSON.stringify({ + arbitrage_opportunities: arbitrageOpportunities, + }), + method: 'POST', + headers: { + 'content-type': 'application/json;charset=UTF-8', + }, + }; - await fetch(WEBHOOK_URL, init); - } catch (err: any) { - console.log(err); + await fetch(WEBHOOK_URL, init); + } catch (err: any) { + console.log(err); + } } + } catch (e) { + console.log('Error at: ', 'sendPriceDiscrepancies'); } } diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 25b6dd88..33d837e5 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -5,33 +5,39 @@ import { AccountBalanceInfos } from '../types/node'; import { extract_group_number_and_address } from './kv'; export async function updateCirculatingSupply(groupNumber: number) { - const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: `grp_${groupNumber}:`, - }); - - console.log( - `found ${cached.keys.length} cached accounts for group ${groupNumber}` - ); - - for (const key of cached.keys) { - const parts = extract_group_number_and_address(key.name); - let addr = parts.address; - let grpN = parts.groupNumber; - - const found = await CIRCULATING_SUPPLY_WATCHLIST.get(`grp_${grpN}:${addr}`); - if (found) { - console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`); - - const account = await updateCachedBalance(addr, grpN); - - if (account !== null) { - console.log( - `updating account (grp_${grpN}:${addr}) balance (${JSON.stringify( - account - )})` - ); + try { + const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ + prefix: `grp_${groupNumber}:`, + }); + + console.log( + `found ${cached.keys.length} cached accounts for group ${groupNumber}` + ); + + for (const key of cached.keys) { + const parts = extract_group_number_and_address(key.name); + let addr = parts.address; + let grpN = parts.groupNumber; + + const found = await CIRCULATING_SUPPLY_WATCHLIST.get( + `grp_${grpN}:${addr}` + ); + if (found) { + console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`); + + const account = await updateCachedBalance(addr, grpN); + + if (account !== null) { + console.log( + `updating account (grp_${grpN}:${addr}) balance (${JSON.stringify( + account + )})` + ); + } } } + } catch (e) { + console.log('Error at: ', 'updateCirculatingSupply'); } } diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 6ee3a2f1..f069a30b 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -5,12 +5,16 @@ import { NodeApi } from '../api/nodeApi'; import { ActiveValidatorsKV } from '../types/kv'; export async function updateActiveValidatorsKV() { - const gql_client = new GraphQLClient(GRAPHQL_API); - const bd_api = new BigDipperApi(gql_client); - const active_validators_resp = await bd_api.get_active_validators(); - - await remove_any_jailed_validators_from_kv(active_validators_resp); - await add_new_active_validators_to_kv(active_validators_resp); + try { + const gql_client = new GraphQLClient(GRAPHQL_API); + const bd_api = new BigDipperApi(gql_client); + const active_validators_resp = await bd_api.get_active_validators(); + + await remove_any_jailed_validators_from_kv(active_validators_resp); + await add_new_active_validators_to_kv(active_validators_resp); + } catch (e) { + console.log('Error at: ', 'updateActiveValidatorsKV'); + } } async function add_new_active_validators_to_kv( @@ -103,13 +107,12 @@ async function put_an_active_validator_in_kv( const data = {} as ActiveValidatorsKV; const node_api = new NodeApi(REST_API); - const delegator_resp = - await node_api.staking_get_delegators_per_validator( - validator_address, - 0, - true, - 1 // set limit param to 1, lessen stress on node api - ); + const delegator_resp = await node_api.staking_get_delegators_per_validator( + validator_address, + 0, + true, + 1 // set limit param to 1, lessen stress on node api + ); data.totalDelegatorsCount = delegator_resp.pagination.total; data.updatedAt = new Date().toUTCString(); @@ -129,9 +132,9 @@ async function delete_stale_validator_from_kv(key: string) { export async function try_getting_delegators_count_from_KV( validator_address: string ) { - const validator_data = await ACTIVE_VALIDATORS.get( + const validator_data = (await ACTIVE_VALIDATORS.get( validator_address - ) as ActiveValidatorsKV; + )) as ActiveValidatorsKV; return validator_data.totalDelegatorsCount ? validator_data.totalDelegatorsCount From 29dfa545d71f41bddf83161921cb1d6a3a1a6682 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 20:02:48 +0300 Subject: [PATCH 171/224] remove voting count --- src/api/bigDipperApi.ts | 13 ++++--------- src/helpers/validators.ts | 26 ++++++++------------------ src/types/bigDipper.ts | 7 ------- src/types/kv.ts | 1 - 4 files changed, 12 insertions(+), 35 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 27cbe664..2e8dbfee 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -1,8 +1,8 @@ import { GraphQLClient } from '../helpers/graphql'; -import { +import { TotalSupplyResponse, TotalStakedCoinsResponse, - ActiveValidatorsResponse + ActiveValidatorsResponse, } from '../types/bigDipper'; export class BigDipperApi { @@ -36,22 +36,17 @@ export class BigDipperApi { data: TotalStakedCoinsResponse; }>(query); return resp.data.staking_pool[0].bonded_tokens; - } + }; get_active_validators = async (): Promise => { const queryActiveValidators = `query ActiveValidators { validator_info(distinct_on: operator_address, where: {validator: {validator_statuses: {jailed: {_eq: false}}}}) { operator_address - validator { - validator_voting_powers { - voting_power - } - } } }`; const activeValidator = await this.graphql_client.query<{ data: ActiveValidatorsResponse; }>(queryActiveValidators); return activeValidator.data; - } + }; } diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index f069a30b..fdd55387 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -38,11 +38,7 @@ async function add_new_active_validators_to_kv( if (!is_active_validator_in_kv) { // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. - put_an_active_validator_in_kv( - latest_active_validator.operator_address, - latest_active_validator.validator.validator_voting_powers[0] - .voting_power - ); + put_an_active_validator_in_kv(latest_active_validator.operator_address); } } } @@ -90,19 +86,13 @@ function create_hashmap_of_validators_addresses_from_api( for (let validator of validators_from_api.validator_info) { if (!hashmap.has(validator.operator_address)) { // since kv contains prefix like grp_1.. we need to extract address only - hashmap.set( - validator.operator_address, - validator.validator.validator_voting_powers[0].voting_power.toString() - ); + hashmap.set(validator.operator_address, validator.operator_address); } } return hashmap; } -async function put_an_active_validator_in_kv( - validator_address: string, - voting_power: number -) { +async function put_an_active_validator_in_kv(validator_address: string) { console.log('putting new active validator', validator_address); const data = {} as ActiveValidatorsKV; const node_api = new NodeApi(REST_API); @@ -114,15 +104,15 @@ async function put_an_active_validator_in_kv( 1 // set limit param to 1, lessen stress on node api ); + console.log('delegator resp', JSON.stringify(delegator_resp)); + data.totalDelegatorsCount = delegator_resp.pagination.total; data.updatedAt = new Date().toUTCString(); - data.votingPower = voting_power.toString(); - - console.log(`Validator data ${data}`); + console.log(`Validator data ${JSON.stringify(data)}`); - const key = `${validator_address}`; + const key = validator_address; await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); - console.log('Added new validator to the list', validator_address); + console.log('Added new validator to the list', key); } async function delete_stale_validator_from_kv(key: string) { await ACTIVE_VALIDATORS.delete(key); diff --git a/src/types/bigDipper.ts b/src/types/bigDipper.ts index 17b8f7d9..3e4a53bb 100644 --- a/src/types/bigDipper.ts +++ b/src/types/bigDipper.ts @@ -23,13 +23,6 @@ export interface ActiveValidatorsResponse { validator_info: [ { operator_address: string; - validator: { - validator_voting_powers: [ - { - voting_power: number; - } - ]; - }; } ]; } diff --git a/src/types/kv.ts b/src/types/kv.ts index 71652abd..f7277895 100644 --- a/src/types/kv.ts +++ b/src/types/kv.ts @@ -1,5 +1,4 @@ export interface ActiveValidatorsKV { totalDelegatorsCount?: string; - votingPower?: string; updatedAt?: string; } From ddc35288abd179e9cec8f74c8822685e211f2203 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 17:12:18 +0000 Subject: [PATCH 172/224] Add circulating supply link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 203e02d7..9c068e30 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ While this figure is available from Cosmos SDK's built-in [`/cosmos/bank/v1beta1 #### Endpoint -`data-api.cheqd.io/supply/circulating` +[`data-api.cheqd.io/supply/circulating`](https://data-api.cheqd.io/supply/circulating) #### Response From 7832642cd2d4e42c74f155df81f32ab10ee7b26d Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 21:08:47 +0300 Subject: [PATCH 173/224] add more logs --- src/api/nodeApi.ts | 15 ++++++++++----- src/helpers/validators.ts | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index ab312cf9..80a327d1 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -56,16 +56,21 @@ export class NodeApi { const pagination_count_total = should_count_total ? 'pagination.count_total=true' : 'pagination.count_total=false'; - const pagination_limit = `pagination.limit=${ - limit ? limit : REST_API_PAGINATION_LIMIT - }`; + const pagination_limit = `pagination.limit=${ + limit ? limit : REST_API_PAGINATION_LIMIT + }`; const pagination_offset = `pagination.offset=${offset}`; // NOTE: be cautious of newlines or spaces. Might make the request URL malformed let resp = await fetch( `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); - console.log(resp.url); - return (await resp.json()) as ValidatorDetailResponse; + + console.log('Response url', resp.url); + const respBody = await resp.json(); + + console.log('Resp body', respBody); + + return respBody as ValidatorDetailResponse; } async staking_get_all_delegations_for_delegator( diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index fdd55387..a2280aa9 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -96,7 +96,7 @@ async function put_an_active_validator_in_kv(validator_address: string) { console.log('putting new active validator', validator_address); const data = {} as ActiveValidatorsKV; const node_api = new NodeApi(REST_API); - + console.log('data', JSON.stringify(data)); const delegator_resp = await node_api.staking_get_delegators_per_validator( validator_address, 0, @@ -114,6 +114,7 @@ async function put_an_active_validator_in_kv(validator_address: string) { await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); console.log('Added new validator to the list', key); } + async function delete_stale_validator_from_kv(key: string) { await ACTIVE_VALIDATORS.delete(key); console.log('Deleted stale validator from the list', key); From 3fef0c2118792fb8a97ecf9cf42e8d14b6599676 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 18:19:56 +0000 Subject: [PATCH 174/224] Remove active validator group binding --- src/bindings.d.ts | 1 - wrangler.toml | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 9b6e49eb..f17de7e5 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -6,7 +6,6 @@ declare global { const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; const ACTIVE_VALIDATORS: KVNamespace; const CIRCULATING_SUPPLY_GROUPS: number; - const ACTIVE_VALIDATOR_GROUPS: number; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; } diff --git a/wrangler.toml b/wrangler.toml index ffbb99f0..487422e8 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -49,9 +49,7 @@ REST_API_PAGINATION_LIMIT = "50" GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" # Number of groups circulating supply watchlist is split into CIRCULATING_SUPPLY_GROUPS = "4" -# Number of groups active validators are split into -ACTIVE_VALIDATOR_GROUPS = "10" -# Moniter market API base url +# Market monitoring API endpoint MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" # The necessary secrets are: @@ -92,7 +90,7 @@ route = { pattern = "data-api-staging.cheqd.io/*", zone_id = "afe3b66243382f2714 # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` -vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", REST_API_PAGINATION_LIMIT = "50", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", ACTIVE_VALIDATOR_GROUPS = "10", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net" } +vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", REST_API_PAGINATION_LIMIT = "50", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net" } # The necessary secrets are: # - WEBHOOK_URL @@ -108,7 +106,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = ["0 * * * *"] +crons = ["0/5 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From af77f20cbc16e2a378c869787343653c0624c547 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 21:20:08 +0300 Subject: [PATCH 175/224] add more logs --- src/api/nodeApi.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 80a327d1..53f57231 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -52,6 +52,10 @@ export class NodeApi { should_count_total: boolean, limit?: number ): Promise { + console.log('address', address); + console.log('limit', limit); + console.log('offset', offset); + // order of query params: count_total -> offset -> limit const pagination_count_total = should_count_total ? 'pagination.count_total=true' @@ -60,10 +64,14 @@ export class NodeApi { limit ? limit : REST_API_PAGINATION_LIMIT }`; const pagination_offset = `pagination.offset=${offset}`; + console.log( + `Before request: ${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` + ); // NOTE: be cautious of newlines or spaces. Might make the request URL malformed let resp = await fetch( `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); + // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 console.log('Response url', resp.url); const respBody = await resp.json(); From 7d4b9357c678a6b4fdaee0fb20272b7f224d2577 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 18:24:09 +0000 Subject: [PATCH 176/224] Update readme for API changes --- README.md | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9c068e30..2249e325 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,7 @@ Cryptocurrency tracking websites such as [CoinMarketCap](https://coinmarketcap.c This figure is _not_ available from any Cosmos SDK API, because the [criteria for determining circulating vs "non-circulating" accounts is defined by CoinMarketCap](https://support.coinmarketcap.com/hc/en-us/articles/360043396252-Supply-Circulating-Total-Max-). -This API calculates the circulating supply by **subtracting** the account balances of a defined list of wallet addresses ("circulating supply watchlist"). Different types of accounts defined in the watchlist are handled as follows: - -1. **Base accounts and Continuous Vesting accounts**: These will always have an entry in BigDipper block explorer, since these accounts have transactions that trigger indexing. -2. **Delayed Vesting accounts**: These accounts present a complex scenario since BigDipper does _not_ index all delayed vesting accounts by default. - 1. **If there have been ANY transactions involving the delayed vesting account**: Delayed vesting accounts can still stake their original vesting allowance, or the account holder may have transferred additional funds into the account. In this scenario, the account _will_ be indexed by BigDipper and the account balance can be fetched via the GraphQL API. - 2. **If there have been NO transactions involving the delayed vesting account**: Delayed vesting accounts with no other transactions beyond the original creation are _not_ indexed by BigDipper. Balances for these accounts are fetched using the standard Cosmos SDK `/cosmos/bank/v1beta1/balances/
` REST API endpoint. +This API calculates the circulating supply by **subtracting** the account balances of a defined list of wallet addresses ("circulating supply watchlist") from the total supply. ### đŸĨŠ Total staked supply @@ -77,7 +72,7 @@ Number of delegators who delegate to a specific validator. #### Rationale -There is no simple Cosmos SDK API to fetch the number of delegators for a given validator. +Running a query to fetch list of delegators per validator can be a computationally-expensive query. This API periodically caches the response. ### 🔐 Vesting Account Balance @@ -188,19 +183,34 @@ While our deployment uses Cloudflare Wrangler, the application itself could be m Wrangler CLI uses [`wrangler.toml` for configuring](https://developers.cloudflare.com/workers/wrangler/configuration/) the application. If you're using this for your own purposes, you will need to replace values for `account_id`, [Cloudflare KV](https://developers.cloudflare.com/workers/learning/how-kv-works/) bindings, `route`, etc. for the application to work correctly along with your own [Cloudflare API tokens](https://developers.cloudflare.com/api/tokens/create). -For the circulating supply API endpoint, Cloudflare Workers will expect to find a Cloudflare KV namespace called `CIRCULATING_SUPPLY_WATCHLIST` with a list of addresses in the `key`. The application _only_ uses the key, so value can be anything. +#### Environment variables + +The application expects these environment variables to be set on Cloudflare: + +1. `TOKEN_EXPONENT`: Denominator for token (default `9` for CHEQ token). +2. `REST_API`: REST API for a Cosmos/cheqd node to target for queries. +3. `REST_API_PAGINATION_LIMIT`: Number of results to fetch in a single query, for queries that require iterating multiple times. (E.g., many account balance queries require this, to be able to get all delegations etc.) +4. `GRAPHQL_API`: GraphQL API for a BigDipper explorer instance for some queries. E.g., the GraphQL API for [cheqd's block explorer](https://explorer.cheqd.io/) is [`https://explorer-gql.cheqd.io/v1/graphql`](https://explorer-gql.cheqd.io/v1/graphql). +5. `CIRCULATING_SUPPLY_GROUPS`: Number of sub-groups the circulating supply watchlist is split into (see sample JSON file below). This is to ensure that any lookups from APIs can be spaced out. +6. `MARKET_MONITORING_API`: Upstream API for running queries from CoinGecko API (see the [market-monitoring repository](https://github.com/cheqd/market-monitoring)). +7. `WEBHOOK_URL`: Zapier webhook URL to send market monitoring data to. Since this is a secret, it's not set in plaintext in `wrangler.toml` but passed via GitHub Actions secrets. + +#### Cloudflare KV bindings + +Cached data for computationally-expensive queries are stored in [Cloudflare KV](https://developers.cloudflare.com/workers/learning/how-kv-works/). -Delayed vesting accounts that have never been involved in a transaction (as described above) should be prefixed with a `delayed:` prefix in the JSON file. Cloudflare allows [filtering KV pair `key`s by prefixes](https://developers.cloudflare.com/workers/runtime-apis/kv/#more-detail) when using a list operation. +1. `CIRCULATING_SUPPLY_WATCHLIST`: This KV is pre-populated with a list of addresses to monitor for circulating supply. Initially, the *value* portion of this can be set to anything, since it will get replaced when [periodic cron triggers](https://developers.cloudflare.com/workers/platform/cron-triggers) run to set the account balance breakdown for this account. In case you have a lot of accounts to monitor, we recommend prefixing the *key* with a `grp_N` prefix which will stagger the API lookup across multiple cron executions. +2. `ACTIVE_VALIDATORS`: List of active validators, fetch from block explorer GraphQL API. When a cron trigger is executed, the total delegator count and update time is stored in this KV. ```jsonc // Sample watchlist JSON file structure [ { - "key": "cheqd1...xxx", - "value": "26-May-2022" // This can be any value + "key": "grp_1:cheqd1...xxx", // Group 1 prefix + "value": "26-May-2022" // This can be any value, and will be updated with account balance breakdown periodically }, { - "key": "delayed:cheqd1...xxx", // This is a delayed account that won't be indexed by BigDipper + "key": "grp_2:cheqd1...xxx", // Group 2 prefix "value": "26-May-2022" } ] From 3ac2d66635cc6d1d980a760f53dacc7416838f92 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Thu, 3 Nov 2022 21:36:02 +0300 Subject: [PATCH 177/224] extract delegator count --- src/api/nodeApi.ts | 8 ++++---- src/helpers/validators.ts | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 53f57231..4b606e2b 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -46,12 +46,12 @@ export class NodeApi { return Number(respJson?.total?.[0]?.amount ?? '0'); } - async staking_get_delegators_per_validator( + async staking_get_delegators_count_per_validator( address: string, offset: number, should_count_total: boolean, limit?: number - ): Promise { + ): Promise { console.log('address', address); console.log('limit', limit); console.log('offset', offset); @@ -74,11 +74,11 @@ export class NodeApi { // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 console.log('Response url', resp.url); - const respBody = await resp.json(); + const respBody = (await resp.json()) as { pagination: { total: string } }; console.log('Resp body', respBody); - return respBody as ValidatorDetailResponse; + return Number(respBody.pagination.total); } async staking_get_all_delegations_for_delegator( diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index a2280aa9..48d9b85d 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -97,16 +97,17 @@ async function put_an_active_validator_in_kv(validator_address: string) { const data = {} as ActiveValidatorsKV; const node_api = new NodeApi(REST_API); console.log('data', JSON.stringify(data)); - const delegator_resp = await node_api.staking_get_delegators_per_validator( - validator_address, - 0, - true, - 1 // set limit param to 1, lessen stress on node api - ); + const delegator_count = + await node_api.staking_get_delegators_count_per_validator( + validator_address, + 0, + true, + 1 // set limit param to 1, lessen stress on node api + ); - console.log('delegator resp', JSON.stringify(delegator_resp)); + console.log('delegator resp', JSON.stringify(delegator_count)); - data.totalDelegatorsCount = delegator_resp.pagination.total; + data.totalDelegatorsCount = delegator_count.toString(); data.updatedAt = new Date().toUTCString(); console.log(`Validator data ${JSON.stringify(data)}`); From 58a42062d493a133a4829c3c3fee6333ab97e379 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 18:50:04 +0000 Subject: [PATCH 178/224] Update nodeApi.ts --- src/api/nodeApi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 4b606e2b..04643750 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -73,7 +73,7 @@ export class NodeApi { ); // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 - console.log('Response url', resp.url); + console.log(JSON.stringify(resp.url)); const respBody = (await resp.json()) as { pagination: { total: string } }; console.log('Resp body', respBody); From c85679dc864f1f9471ee1d25d923d7a770899060 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 18:54:54 +0000 Subject: [PATCH 179/224] Trying another logging method --- src/api/nodeApi.ts | 2 +- wrangler.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 04643750..a4a72f21 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -73,7 +73,7 @@ export class NodeApi { ); // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 - console.log(JSON.stringify(resp.url)); + console.log(JSON.stringify([...resp.headers])); const respBody = (await resp.json()) as { pagination: { total: string } }; console.log('Resp body', respBody); diff --git a/wrangler.toml b/wrangler.toml index 487422e8..dcad3b45 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -106,7 +106,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = ["0/5 * * * *"] +crons = ["0/3 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From f855d7559d5a776feae24fa892cd119615be6257 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Thu, 3 Nov 2022 19:06:11 +0000 Subject: [PATCH 180/224] Logging using map --- src/api/nodeApi.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index a4a72f21..38834a4f 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -52,9 +52,7 @@ export class NodeApi { should_count_total: boolean, limit?: number ): Promise { - console.log('address', address); - console.log('limit', limit); - console.log('offset', offset); + console.log('address', address, 'limit', limit, 'offset', offset); // order of query params: count_total -> offset -> limit const pagination_count_total = should_count_total @@ -73,7 +71,7 @@ export class NodeApi { ); // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 - console.log(JSON.stringify([...resp.headers])); + console.log(new Map(resp.headers)); const respBody = (await resp.json()) as { pagination: { total: string } }; console.log('Resp body', respBody); From 076fcaacab5398cd896074e3560dedfa9e1c2212 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:13:18 +0300 Subject: [PATCH 181/224] try catch api call --- src/api/nodeApi.ts | 57 ++++++++++++++++++--------------- src/handlers/webhookTriggers.ts | 2 +- src/helpers/validators.ts | 3 +- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 38834a4f..d296fb7b 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -51,32 +51,37 @@ export class NodeApi { offset: number, should_count_total: boolean, limit?: number - ): Promise { - console.log('address', address, 'limit', limit, 'offset', offset); - - // order of query params: count_total -> offset -> limit - const pagination_count_total = should_count_total - ? 'pagination.count_total=true' - : 'pagination.count_total=false'; - const pagination_limit = `pagination.limit=${ - limit ? limit : REST_API_PAGINATION_LIMIT - }`; - const pagination_offset = `pagination.offset=${offset}`; - console.log( - `Before request: ${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` - ); - // NOTE: be cautious of newlines or spaces. Might make the request URL malformed - let resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` - ); - // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 - - console.log(new Map(resp.headers)); - const respBody = (await resp.json()) as { pagination: { total: string } }; - - console.log('Resp body', respBody); - - return Number(respBody.pagination.total); + ): Promise { + try { + console.log('address', address, 'limit', limit, 'offset', offset); + + // order of query params: count_total -> offset -> limit + const pagination_count_total = should_count_total + ? 'pagination.count_total=true' + : 'pagination.count_total=false'; + const pagination_limit = `pagination.limit=${ + limit ? limit : REST_API_PAGINATION_LIMIT + }`; + const pagination_offset = `pagination.offset=${offset}`; + console.log( + `Before request: ${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` + ); + // NOTE: be cautious of newlines or spaces. Might make the request URL malformed + let resp = await fetch( + `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` + ); + // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 + + console.log(new Map(resp.headers)); + const respBody = (await resp.json()) as { pagination: { total: string } }; + + console.log('Resp body', respBody); + + return Number(respBody.pagination.total); + } catch (e) { + console.log('Error', e); + return null; + } } async staking_get_all_delegations_for_delegator( diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 20e7117c..751faa5d 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -4,7 +4,7 @@ import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: Event) { console.log('Triggering webhook...'); - await sendPriceDiscrepancies(); // removed to test + await sendPriceDiscrepancies(); await updateCirculatingSupply( getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 48d9b85d..6bff7101 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -107,7 +107,8 @@ async function put_an_active_validator_in_kv(validator_address: string) { console.log('delegator resp', JSON.stringify(delegator_count)); - data.totalDelegatorsCount = delegator_count.toString(); + data.totalDelegatorsCount = + delegator_count === null ? 'null' : delegator_count.toString(); data.updatedAt = new Date().toUTCString(); console.log(`Validator data ${JSON.stringify(data)}`); From ca8f9cf007dd4774d413845478521f584e6babaa Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:13:34 +0300 Subject: [PATCH 182/224] remove unused type --- src/types/node.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/types/node.ts b/src/types/node.ts index 59225091..a8f3c2d5 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -42,14 +42,6 @@ export class Delegation { } } -export interface ValidatorDelegationsCountResponse { - delegations: { - pagination: { - total: number; - }; - }; -} - export interface ValidatorDetailResponse { delegation_responses: [ { @@ -121,6 +113,6 @@ export interface UnbondingResponse { } export interface RewardsResponse { - rewards: Record[]; - total: Coin[]; + rewards: Record[]; + total: Coin[]; } From f036d155597eeac218a79894165558dc4cca6934 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:29:11 +0300 Subject: [PATCH 183/224] add a additional cron for testing --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index dcad3b45..3f138c79 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -106,7 +106,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = ["0/3 * * * *"] +crons = ["0/3 * * * *", "0/1 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 2a7cf512707e5c02ff6888e76be3d96428907f21 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:29:55 +0300 Subject: [PATCH 184/224] run updating active validators kv on separate cron --- src/handlers/webhookTriggers.ts | 6 ++++-- src/index.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 751faa5d..08bfbc5c 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -2,14 +2,16 @@ import { updateCirculatingSupply } from '../helpers/circulating'; import { updateActiveValidatorsKV } from '../helpers/validators'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; -export async function webhookTriggers(event: Event) { +export async function webhookTriggers(event: ScheduledEvent) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); await updateCirculatingSupply( getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) ); - await updateActiveValidatorsKV(); + if (event.cron === '*/1 * * * *') { + await updateActiveValidatorsKV(); + } } export async function sendPriceDiscrepancies() { diff --git a/src/index.ts b/src/index.ts index 355880d0..2ad35c05 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import { handler as allArbitrageOpportunitiesHandler } from './handlers/allArbit import { handler as arbitrageOpportunitiesHandler } from './handlers/arbitrageOpportunities'; import { webhookTriggers } from './handlers/webhookTriggers'; -addEventListener('scheduled', (event: any) => { +addEventListener('scheduled', (event: ScheduledEvent) => { event.waitUntil(webhookTriggers(event)); }); From e83a309ac219ffd8073116c8933b2db735d95950 Mon Sep 17 00:00:00 2001 From: Benyam Seifu <45682486+benyam7@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:30:24 +0300 Subject: [PATCH 185/224] put only one active validator on kv to lessen load --- src/helpers/validators.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 6bff7101..d93b9a3f 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -39,6 +39,8 @@ async function add_new_active_validators_to_kv( if (!is_active_validator_in_kv) { // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. put_an_active_validator_in_kv(latest_active_validator.operator_address); + // break after puttin first validator: TODO just for testing Remove this + break; } } } From aa5174d1df2139642efbf0639ba320c0f2c70dfe Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 12:16:53 +0000 Subject: [PATCH 186/224] Disable market monitoring and circulating supply triggers --- src/handlers/webhookTriggers.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 08bfbc5c..a429aa89 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -4,14 +4,14 @@ import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: ScheduledEvent) { console.log('Triggering webhook...'); - await sendPriceDiscrepancies(); + // await sendPriceDiscrepancies(); - await updateCirculatingSupply( +/* await updateCirculatingSupply( getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) - ); - if (event.cron === '*/1 * * * *') { + ); */ + //if (event.cron === '*/1 * * * *') { await updateActiveValidatorsKV(); - } + //} } export async function sendPriceDiscrepancies() { From ba5c115988723148e4ce35d30153afa80bad2088 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 12:18:45 +0000 Subject: [PATCH 187/224] Set triggrer to 2 mins --- wrangler.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrangler.toml b/wrangler.toml index 3f138c79..018e975b 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -106,7 +106,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = ["0/3 * * * *", "0/1 * * * *"] +crons = [ "0/2 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 282f03b575c451cc7eb2b1d21c0bf5cf6ea6ad66 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 12:22:17 +0000 Subject: [PATCH 188/224] Deactivate adding validators --- src/helpers/validators.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index d93b9a3f..2c3b0d8d 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -11,7 +11,7 @@ export async function updateActiveValidatorsKV() { const active_validators_resp = await bd_api.get_active_validators(); await remove_any_jailed_validators_from_kv(active_validators_resp); - await add_new_active_validators_to_kv(active_validators_resp); + //await add_new_active_validators_to_kv(active_validators_resp); } catch (e) { console.log('Error at: ', 'updateActiveValidatorsKV'); } From fdded4c00bc46f3e91d270052943f6ae9d801326 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 12:25:39 +0000 Subject: [PATCH 189/224] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e5de2ef8..01a3e171 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### APP-SPECIFIC EXCLUSIONS ### wrangler.dev.toml watchlist.json -watchlist-old.json +watchlist-*.json ### GENERAL EXCLUSIONS ### From 707492619757245de8bb287cce273b65976cff75 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 13:54:41 +0000 Subject: [PATCH 190/224] Log delete stale --- .github/workflows/dispatch.yml | 6 +++--- src/helpers/validators.ts | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 3c08a54d..5c9e4ea9 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -6,9 +6,9 @@ concurrency: jobs: - call-lint: - name: "Lint" - uses: ./.github/workflows/lint.yml + # call-lint: + # name: "Lint" + # uses: ./.github/workflows/lint.yml call-deploy: name: "Deploy" diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 2c3b0d8d..4bf7a1c1 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -50,13 +50,14 @@ async function remove_any_jailed_validators_from_kv( ) { // Loop through validators from kv, and if // a validator from KV doesn't exist in API remove from KV - console.log('Removing jailed validators, if any'); + console.log('Execute remove_any_jailed_validators_from_kv'); const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); const active_validators_from_api_hash_map = create_hashmap_of_validators_addresses_from_api(active_validators); for (let validator_from_kv of active_validators_from_kv.keys) { if (!active_validators_from_api_hash_map.has(validator_from_kv.name)) { + console.log("Calling delete_stale_validator_from_kv", validator_from_kv.name); delete_stale_validator_from_kv(validator_from_kv.name); } } From 8f4fb0cc55d1b23433404b4799410a79e0198f74 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 13:59:43 +0000 Subject: [PATCH 191/224] Update dispatch.yml --- .github/workflows/dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 5c9e4ea9..cd91f4d6 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,6 +12,6 @@ jobs: call-deploy: name: "Deploy" - needs: call-lint + # needs: call-lint uses: ./.github/workflows/deploy.yml secrets: inherit From 81798f369aa567185bfe32d9fb6571e44edfa180 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 14:08:01 +0000 Subject: [PATCH 192/224] Update validators.ts --- src/helpers/validators.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts index 4bf7a1c1..455228b2 100644 --- a/src/helpers/validators.ts +++ b/src/helpers/validators.ts @@ -38,7 +38,7 @@ async function add_new_active_validators_to_kv( if (!is_active_validator_in_kv) { // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. - put_an_active_validator_in_kv(latest_active_validator.operator_address); + await put_an_active_validator_in_kv(latest_active_validator.operator_address); // break after puttin first validator: TODO just for testing Remove this break; } @@ -58,7 +58,7 @@ async function remove_any_jailed_validators_from_kv( for (let validator_from_kv of active_validators_from_kv.keys) { if (!active_validators_from_api_hash_map.has(validator_from_kv.name)) { console.log("Calling delete_stale_validator_from_kv", validator_from_kv.name); - delete_stale_validator_from_kv(validator_from_kv.name); + await delete_stale_validator_from_kv(validator_from_kv.name); } } } From a60edc6bf6168dba9f1595a8be197ea9b644cba1 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 15:18:21 +0000 Subject: [PATCH 193/224] Delete validators helper --- src/handlers/webhookTriggers.ts | 6 +- src/helpers/validators.ts | 138 -------------------------------- 2 files changed, 1 insertion(+), 143 deletions(-) delete mode 100644 src/helpers/validators.ts diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index a429aa89..38c33637 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -1,17 +1,13 @@ import { updateCirculatingSupply } from '../helpers/circulating'; -import { updateActiveValidatorsKV } from '../helpers/validators'; import { filterArbitrageOpportunities } from './arbitrageOpportunities'; export async function webhookTriggers(event: ScheduledEvent) { console.log('Triggering webhook...'); - // await sendPriceDiscrepancies(); + await sendPriceDiscrepancies(); /* await updateCirculatingSupply( getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) ); */ - //if (event.cron === '*/1 * * * *') { - await updateActiveValidatorsKV(); - //} } export async function sendPriceDiscrepancies() { diff --git a/src/helpers/validators.ts b/src/helpers/validators.ts deleted file mode 100644 index 455228b2..00000000 --- a/src/helpers/validators.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { BigDipperApi } from '../api/bigDipperApi'; -import { ActiveValidatorsResponse } from '../types/bigDipper'; -import { GraphQLClient } from './graphql'; -import { NodeApi } from '../api/nodeApi'; -import { ActiveValidatorsKV } from '../types/kv'; - -export async function updateActiveValidatorsKV() { - try { - const gql_client = new GraphQLClient(GRAPHQL_API); - const bd_api = new BigDipperApi(gql_client); - const active_validators_resp = await bd_api.get_active_validators(); - - await remove_any_jailed_validators_from_kv(active_validators_resp); - //await add_new_active_validators_to_kv(active_validators_resp); - } catch (e) { - console.log('Error at: ', 'updateActiveValidatorsKV'); - } -} - -async function add_new_active_validators_to_kv( - active_validators: ActiveValidatorsResponse -) { - console.log('Adding new active validators to KV, if any...'); - - const latest_active_validators_from_api = active_validators.validator_info; - const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); - const active_validators_from_kv_hashmap = - create_hashmap_of_validators_addresses_from_kv( - active_validators_from_kv.keys as KVNamespaceListKey[] - ); - - for (let latest_active_validator of latest_active_validators_from_api) { - // if latest_active_validator is in kv, keep it. - // if latest_active_validator is not in kv, add it. - const is_active_validator_in_kv = active_validators_from_kv_hashmap.has( - latest_active_validator.operator_address - ); - - if (!is_active_validator_in_kv) { - // can only update validator's voting power. it's delegator count is updated when TOTAL_DELEGATORS KV is updated. - await put_an_active_validator_in_kv(latest_active_validator.operator_address); - // break after puttin first validator: TODO just for testing Remove this - break; - } - } -} - -async function remove_any_jailed_validators_from_kv( - active_validators: ActiveValidatorsResponse -) { - // Loop through validators from kv, and if - // a validator from KV doesn't exist in API remove from KV - console.log('Execute remove_any_jailed_validators_from_kv'); - const active_validators_from_kv = await ACTIVE_VALIDATORS.list(); - const active_validators_from_api_hash_map = - create_hashmap_of_validators_addresses_from_api(active_validators); - - for (let validator_from_kv of active_validators_from_kv.keys) { - if (!active_validators_from_api_hash_map.has(validator_from_kv.name)) { - console.log("Calling delete_stale_validator_from_kv", validator_from_kv.name); - await delete_stale_validator_from_kv(validator_from_kv.name); - } - } -} - -function create_hashmap_of_validators_addresses_from_kv( - validators_from_KV?: KVNamespaceListKey[] -): Map { - const hashmap = new Map(); - // In case keys contain prefixes, since they are from KV - if (!validators_from_KV) { - return hashmap; - } - - for (let key of validators_from_KV) { - if (!hashmap.has(key.name)) { - // since kv contains prefix like grp_1.. we need to extract address only - hashmap.set(key.name, key.name); - } - } - return hashmap; -} - -function create_hashmap_of_validators_addresses_from_api( - validators_from_api: ActiveValidatorsResponse -): Map { - // In case keys contain prefixes, since they are from KV - const hashmap = new Map(); - for (let validator of validators_from_api.validator_info) { - if (!hashmap.has(validator.operator_address)) { - // since kv contains prefix like grp_1.. we need to extract address only - hashmap.set(validator.operator_address, validator.operator_address); - } - } - return hashmap; -} - -async function put_an_active_validator_in_kv(validator_address: string) { - console.log('putting new active validator', validator_address); - const data = {} as ActiveValidatorsKV; - const node_api = new NodeApi(REST_API); - console.log('data', JSON.stringify(data)); - const delegator_count = - await node_api.staking_get_delegators_count_per_validator( - validator_address, - 0, - true, - 1 // set limit param to 1, lessen stress on node api - ); - - console.log('delegator resp', JSON.stringify(delegator_count)); - - data.totalDelegatorsCount = - delegator_count === null ? 'null' : delegator_count.toString(); - data.updatedAt = new Date().toUTCString(); - console.log(`Validator data ${JSON.stringify(data)}`); - - const key = validator_address; - await ACTIVE_VALIDATORS.put(key, JSON.stringify(data)); - console.log('Added new validator to the list', key); -} - -async function delete_stale_validator_from_kv(key: string) { - await ACTIVE_VALIDATORS.delete(key); - console.log('Deleted stale validator from the list', key); -} - -export async function try_getting_delegators_count_from_KV( - validator_address: string -) { - const validator_data = (await ACTIVE_VALIDATORS.get( - validator_address - )) as ActiveValidatorsKV; - - return validator_data.totalDelegatorsCount - ? validator_data.totalDelegatorsCount - : null; -} From c132d79dacbcd3e7ea87930742e3f9f89a4b231c Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 15:58:14 +0000 Subject: [PATCH 194/224] Update circulating supply groups --- wrangler.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index 018e975b..cca290ba 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -33,7 +33,7 @@ route = { pattern = "data-api.cheqd.io/*", zone_id = "afe3b66243382f27140e6feeaa # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "a9bc7aaa54ee4394ae6b9abe43e05ad6" }, { binding = "ACTIVE_VALIDATORS", id = "b9afd59ab39b4609905775bd651efd7f" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "270dbd79fa434edf9174fac7d5bc1bdf" } ] # Map of environment variables to set when deploying the Worker @@ -48,7 +48,7 @@ REST_API_PAGINATION_LIMIT = "50" # GraphQL API endpoint for target network. Must be sourced from a BigDipper instance. GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql" # Number of groups circulating supply watchlist is split into -CIRCULATING_SUPPLY_GROUPS = "4" +CIRCULATING_SUPPLY_GROUPS = "24" # Market monitoring API endpoint MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" @@ -57,7 +57,7 @@ MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" # Run `echo | wrangler secret put ` for each of these [triggers] -crons = ["0 * * * *"] +crons = ["0/10 * * * *"] ############################################################### ### SECTION 3: Local Development ### @@ -90,7 +90,7 @@ route = { pattern = "data-api-staging.cheqd.io/*", zone_id = "afe3b66243382f2714 # Map of environment variables to set when deploying the Worker # Not inherited. @default `{}` -vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", REST_API_PAGINATION_LIMIT = "50", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "4", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net" } +vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api.cheqd.net", REST_API_PAGINATION_LIMIT = "50", GRAPHQL_API = "https://explorer-gql.cheqd.io/v1/graphql", CIRCULATING_SUPPLY_GROUPS = "24", MARKET_MONITORING_API = "https://market-monitoring-staging.cheqd.net" } # The necessary secrets are: # - WEBHOOK_URL @@ -101,12 +101,12 @@ vars = { ENVIRONMENT = "staging", TOKEN_EXPONENT = "9", REST_API = "https://api. # @default `[]` kv_namespaces = [ - { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "86891d184f7f40ee9b403a94a76fcdab" }, { binding = "ACTIVE_VALIDATORS", id = "d78a08202b1941bcb24413d3a2891d12" } + { binding = "CIRCULATING_SUPPLY_WATCHLIST", id = "83699afe22654413ae141bb70d37554d" } ] # Cron triggers for staging worker [env.staging.triggers] -crons = [ "0/2 * * * *"] +crons = [ "0/5 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From dacf70b9fd105da6496f58f33b7e85804419d2c7 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 15:59:08 +0000 Subject: [PATCH 195/224] Activate circulating supply trigger --- src/handlers/webhookTriggers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/handlers/webhookTriggers.ts b/src/handlers/webhookTriggers.ts index 38c33637..d54893e2 100644 --- a/src/handlers/webhookTriggers.ts +++ b/src/handlers/webhookTriggers.ts @@ -5,9 +5,9 @@ export async function webhookTriggers(event: ScheduledEvent) { console.log('Triggering webhook...'); await sendPriceDiscrepancies(); -/* await updateCirculatingSupply( + await updateCirculatingSupply( getRandomGroup(Number(CIRCULATING_SUPPLY_GROUPS)) - ); */ + ); } export async function sendPriceDiscrepancies() { From 51c1433d97c0b4f129a2a90a0a905065614460fc Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 15:59:18 +0000 Subject: [PATCH 196/224] Update group prefix --- README.md | 6 +++--- src/helpers/circulating.ts | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2249e325..de26f85e 100644 --- a/README.md +++ b/README.md @@ -199,18 +199,18 @@ The application expects these environment variables to be set on Cloudflare: Cached data for computationally-expensive queries are stored in [Cloudflare KV](https://developers.cloudflare.com/workers/learning/how-kv-works/). -1. `CIRCULATING_SUPPLY_WATCHLIST`: This KV is pre-populated with a list of addresses to monitor for circulating supply. Initially, the *value* portion of this can be set to anything, since it will get replaced when [periodic cron triggers](https://developers.cloudflare.com/workers/platform/cron-triggers) run to set the account balance breakdown for this account. In case you have a lot of accounts to monitor, we recommend prefixing the *key* with a `grp_N` prefix which will stagger the API lookup across multiple cron executions. +1. `CIRCULATING_SUPPLY_WATCHLIST`: This KV is pre-populated with a list of addresses to monitor for circulating supply. Initially, the *value* portion of this can be set to anything, since it will get replaced when [periodic cron triggers](https://developers.cloudflare.com/workers/platform/cron-triggers) run to set the account balance breakdown for this account. In case you have a lot of accounts to monitor, we recommend prefixing the *key* with a `group_N` prefix which will stagger the API lookup across multiple cron executions. 2. `ACTIVE_VALIDATORS`: List of active validators, fetch from block explorer GraphQL API. When a cron trigger is executed, the total delegator count and update time is stored in this KV. ```jsonc // Sample watchlist JSON file structure [ { - "key": "grp_1:cheqd1...xxx", // Group 1 prefix + "key": "group_1:cheqd1...xxx", // Group 1 prefix "value": "26-May-2022" // This can be any value, and will be updated with account balance breakdown periodically }, { - "key": "grp_2:cheqd1...xxx", // Group 2 prefix + "key": "group_2:cheqd1...xxx", // Group 2 prefix "value": "26-May-2022" } ] diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 33d837e5..0b723768 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -7,7 +7,7 @@ import { extract_group_number_and_address } from './kv'; export async function updateCirculatingSupply(groupNumber: number) { try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list({ - prefix: `grp_${groupNumber}:`, + prefix: `group_${groupNumber}:`, }); console.log( @@ -20,7 +20,7 @@ export async function updateCirculatingSupply(groupNumber: number) { let grpN = parts.groupNumber; const found = await CIRCULATING_SUPPLY_WATCHLIST.get( - `grp_${grpN}:${addr}` + `group_${grpN}:${addr}` ); if (found) { console.log(`found ${key.name} (addr=${addr}) grp=${grpN}`); @@ -29,7 +29,7 @@ export async function updateCirculatingSupply(groupNumber: number) { if (account !== null) { console.log( - `updating account (grp_${grpN}:${addr}) balance (${JSON.stringify( + `updating account (group_${grpN}:${addr}) balance (${JSON.stringify( account )})` ); @@ -49,7 +49,7 @@ export async function updateCachedBalance(addr: string, grpN: number) { const data = JSON.stringify(account_balance_infos); - await CIRCULATING_SUPPLY_WATCHLIST.put(`grp_${grpN}:${addr}`, data); + await CIRCULATING_SUPPLY_WATCHLIST.put(`group_${grpN}:${addr}`, data); console.log(`account "${addr}" balance updated. (${data})`); } catch (e: any) { From 917ecd5e48a129724954409163c9a1fddc486a0d Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 16:03:55 +0000 Subject: [PATCH 197/224] Remove unused functions in nodeApi --- src/api/nodeApi.ts | 51 +--------------------------------------------- 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index d296fb7b..88da88b7 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -3,8 +3,7 @@ import { Coin, DelegationsResponse, UnbondingResponse, - RewardsResponse, - ValidatorDetailResponse, + RewardsResponse } from '../types/node'; export class NodeApi { @@ -46,44 +45,6 @@ export class NodeApi { return Number(respJson?.total?.[0]?.amount ?? '0'); } - async staking_get_delegators_count_per_validator( - address: string, - offset: number, - should_count_total: boolean, - limit?: number - ): Promise { - try { - console.log('address', address, 'limit', limit, 'offset', offset); - - // order of query params: count_total -> offset -> limit - const pagination_count_total = should_count_total - ? 'pagination.count_total=true' - : 'pagination.count_total=false'; - const pagination_limit = `pagination.limit=${ - limit ? limit : REST_API_PAGINATION_LIMIT - }`; - const pagination_offset = `pagination.offset=${offset}`; - console.log( - `Before request: ${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` - ); - // NOTE: be cautious of newlines or spaces. Might make the request URL malformed - let resp = await fetch( - `${this.base_rest_api_url}/cosmos/staking/v1beta1/validators/${address}/delegations?${pagination_count_total}&${pagination_limit}&${pagination_offset}` - ); - // https://api.cheqd.net/cosmos/staking/v1beta1/validators/cheqdvaloper1nxlprsp26qyjarp8c6mjf33rxvh7mll7uy5zhk/delegations?pagination.count_total=true&pagination.limit=1&pagination.offset=0 - - console.log(new Map(resp.headers)); - const respBody = (await resp.json()) as { pagination: { total: string } }; - - console.log('Resp body', respBody); - - return Number(respBody.pagination.total); - } catch (e) { - console.log('Error', e); - return null; - } - } - async staking_get_all_delegations_for_delegator( address: string, offset: number, @@ -103,8 +64,6 @@ export class NodeApi { `${this.base_rest_api_url}/cosmos/staking/v1beta1/delegations/${address}?${pagination_count_total}&${pagination_limit}&${pagination_offset}` ); - console.log(`Response status for delegator count ${resp.status}`); - return (await resp.json()) as DelegationsResponse; } @@ -126,12 +85,4 @@ export class NodeApi { return (await resp.json()) as UnbondingResponse; } - - async get_latest_block_height(): Promise { - const resp = await fetch(`${this.base_rest_api_url}/blocks/latest`); - let respJson = (await resp.json()) as { - block: { header: { height: number } }; - }; - return Number(respJson.block.header.height); - } } From f11bac5ad9f441f41712631ca42ed626d5abda72 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 16:06:41 +0000 Subject: [PATCH 198/224] Removed unused BigDipper API --- src/api/bigDipperApi.ts | 16 ++-------------- src/api/nodeApi.ts | 2 +- src/handlers/totalStakedCoins.ts | 2 +- src/handlers/totalSupply.ts | 2 +- src/helpers/circulating.ts | 6 +++--- 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/src/api/bigDipperApi.ts b/src/api/bigDipperApi.ts index 2e8dbfee..83b41126 100644 --- a/src/api/bigDipperApi.ts +++ b/src/api/bigDipperApi.ts @@ -8,7 +8,7 @@ import { export class BigDipperApi { constructor(public readonly graphql_client: GraphQLClient) {} - async get_total_supply(): Promise { + async getTotalSupply(): Promise { let query = `query TotalSupply { supply { coins @@ -25,7 +25,7 @@ export class BigDipperApi { ); } - get_total_staked_coins = async (): Promise => { + getTotalStakedCoins = async (): Promise => { let query = `query StakingInfo{ staking_pool { bonded_tokens @@ -37,16 +37,4 @@ export class BigDipperApi { }>(query); return resp.data.staking_pool[0].bonded_tokens; }; - - get_active_validators = async (): Promise => { - const queryActiveValidators = `query ActiveValidators { - validator_info(distinct_on: operator_address, where: {validator: {validator_statuses: {jailed: {_eq: false}}}}) { - operator_address - } - }`; - const activeValidator = await this.graphql_client.query<{ - data: ActiveValidatorsResponse; - }>(queryActiveValidators); - return activeValidator.data; - }; } diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index 88da88b7..ce9c5b7b 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -9,7 +9,7 @@ import { export class NodeApi { constructor(public readonly base_rest_api_url: string) {} - async bank_get_total_supply_ncheq(): Promise { + async bank_getTotalSupply_ncheq(): Promise { let resp = await fetch( `${this.base_rest_api_url}/cosmos/bank/v1beta1/supply/ncheq` ); diff --git a/src/handlers/totalStakedCoins.ts b/src/handlers/totalStakedCoins.ts index c4ffd105..25ddd23d 100644 --- a/src/handlers/totalStakedCoins.ts +++ b/src/handlers/totalStakedCoins.ts @@ -7,7 +7,7 @@ export async function handler(request: Request): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - let total_staked_coins = await bd_api.get_total_staked_coins(); + let total_staked_coins = await bd_api.getTotalStakedCoins(); return new Response(ncheq_to_cheq_fixed(Number(total_staked_coins))); } diff --git a/src/handlers/totalSupply.ts b/src/handlers/totalSupply.ts index b996aa90..e5c1c975 100644 --- a/src/handlers/totalSupply.ts +++ b/src/handlers/totalSupply.ts @@ -6,6 +6,6 @@ import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); - const total_supply = await bd_api.get_total_supply(); + const total_supply = await bd_api.getTotalSupply(); return new Response(ncheq_to_cheq_fixed(total_supply)); } diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 0b723768..a8a91cdc 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -57,16 +57,16 @@ export async function updateCachedBalance(addr: string, grpN: number) { } } -export async function get_total_supply(): Promise { +export async function getTotalSupply(): Promise { let node_api = new NodeApi(REST_API); - let total_supply_ncheq = await node_api.bank_get_total_supply_ncheq(); + let total_supply_ncheq = await node_api.bank_getTotalSupply_ncheq(); const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); return total_supply; } export async function get_circulating_supply(): Promise { - const total_supply = await get_total_supply(); + const total_supply = await getTotalSupply(); try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); From 942dbb4e7e6335ffad6b52f4b5eeec16c3cb8a0b Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 16:07:56 +0000 Subject: [PATCH 199/224] Change market monitoring functions to PascalCase --- src/api/marketMonitorApi.ts | 2 +- src/handlers/allArbitrageOpportunities.ts | 2 +- src/handlers/arbitrageOpportunities.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/marketMonitorApi.ts b/src/api/marketMonitorApi.ts index 28ebcc49..82300a55 100644 --- a/src/api/marketMonitorApi.ts +++ b/src/api/marketMonitorApi.ts @@ -2,7 +2,7 @@ import { MarketMonitorData } from '../types/marketMonitor'; export class MarketMonitorApi { constructor(public readonly base_market_monitor_api_url: string) {} - async get_market_monitor_data(): Promise { + async getMarketMonitoringData(): Promise { const requestOptions = { method: 'GET', }; diff --git a/src/handlers/allArbitrageOpportunities.ts b/src/handlers/allArbitrageOpportunities.ts index 76badc5b..6d1177f0 100644 --- a/src/handlers/allArbitrageOpportunities.ts +++ b/src/handlers/allArbitrageOpportunities.ts @@ -4,7 +4,7 @@ export async function fetchPrices() { let market_monitor_api = new MarketMonitorApi( `${MARKET_MONITORING_API}` ); - return await market_monitor_api.get_market_monitor_data(); + return await market_monitor_api.getMarketMonitoringData(); } export async function handler(request: Request): Promise { const payload = await fetchPrices(); diff --git a/src/handlers/arbitrageOpportunities.ts b/src/handlers/arbitrageOpportunities.ts index 2f89204b..bc6cdede 100644 --- a/src/handlers/arbitrageOpportunities.ts +++ b/src/handlers/arbitrageOpportunities.ts @@ -5,7 +5,7 @@ async function fetchPrices() { let market_monitor_api = new MarketMonitorApi( `${MARKET_MONITORING_API}` ); - return await market_monitor_api.get_market_monitor_data(); + return await market_monitor_api.getMarketMonitoringData(); } export async function filterArbitrageOpportunities(): Promise< From 4da6e544153216a882034e2541b8c35789872a4a Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 16:28:21 +0000 Subject: [PATCH 200/224] Set node API methods to camelCase --- src/api/nodeApi.ts | 19 +++++-------------- src/handlers/liquidBalance.ts | 10 +++++----- src/handlers/vestedBalance.ts | 2 +- src/handlers/vestingBalance.ts | 2 +- src/helpers/balance.ts | 12 ++++++------ src/helpers/circulating.ts | 15 ++++++--------- 6 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/api/nodeApi.ts b/src/api/nodeApi.ts index ce9c5b7b..4c05d623 100644 --- a/src/api/nodeApi.ts +++ b/src/api/nodeApi.ts @@ -9,16 +9,7 @@ import { export class NodeApi { constructor(public readonly base_rest_api_url: string) {} - async bank_getTotalSupply_ncheq(): Promise { - let resp = await fetch( - `${this.base_rest_api_url}/cosmos/bank/v1beta1/supply/ncheq` - ); - let respJson = (await resp.json()) as { amount: { amount: number } }; - - return respJson.amount.amount; - } - - async auth_get_account(address: string): Promise { + async getAccountInfo(address: string): Promise { let resp = await fetch( `${this.base_rest_api_url}/cosmos/auth/v1beta1/accounts/${address}` ); @@ -27,7 +18,7 @@ export class NodeApi { return respJson.account; } - async bank_get_account_balances(address: string): Promise { + async getAvailableBalance(address: string): Promise { let resp = await fetch( `${this.base_rest_api_url}/cosmos/bank/v1beta1/balances/${address}` ); @@ -36,7 +27,7 @@ export class NodeApi { return respJson.balances; } - async distribution_get_total_rewards(address: string): Promise { + async distributionGetRewards(address: string): Promise { let resp = await fetch( `${this.base_rest_api_url}/cosmos/distribution/v1beta1/delegators/${address}/rewards` ); @@ -45,7 +36,7 @@ export class NodeApi { return Number(respJson?.total?.[0]?.amount ?? '0'); } - async staking_get_all_delegations_for_delegator( + async getAllDelegations( address: string, offset: number, should_count_total: boolean, @@ -67,7 +58,7 @@ export class NodeApi { return (await resp.json()) as DelegationsResponse; } - async staking_get_all_unbonding_delegations_for_delegator( + async getAllUnbondingDelegations( address: string, offset: number, should_count_total: boolean diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index a944ff8e..53112083 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -12,23 +12,23 @@ export async function handler(request: Request): Promise { } let api = new NodeApi(REST_API); - const account = await api.auth_get_account(address) + const account = await api.getAccountInfo(address) if (!is_vesting_account_type(account["@type"])) { throw new Error(`Only vesting accounts are supported. Accounts type '${account["@type"]}'.`) } if(is_delayed_vesting_account_type(account?.["@type"])) { - let balance = account?.base_vesting_account?.base_account?.sequence !== '0' ? Number(await (await api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0') : 0; - let rewards = Number(await (await api.distribution_get_total_rewards(address)) ?? '0'); + let balance = account?.base_vesting_account?.base_account?.sequence !== '0' ? Number(await (await api.getAvailableBalance(address)).find(b => b.denom === "ncheq")?.amount ?? '0') : 0; + let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let delegated = Number(account?.base_vesting_account?.delegated_free?.find(d => d.denom === "ncheq")?.amount ?? '0'); return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); } let vested_coins = calculate_vested_coins(account); - let balance = Number(await (await api.bank_get_account_balances(address)).find(b => b.denom === "ncheq")?.amount ?? '0') - let rewards = Number(await (await api.distribution_get_total_rewards(address)) ?? '0'); + let balance = Number(await (await api.getAvailableBalance(address)).find(b => b.denom === "ncheq")?.amount ?? '0') + let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let liquid_coins = vested_coins + balance + rewards; return new Response(ncheq_to_cheq_fixed(liquid_coins)); diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index 7cf60b26..eea79b11 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -15,7 +15,7 @@ export async function handler(request: Request): Promise { } let api = new NodeApi(REST_API); - const account = await api.auth_get_account(address); + const account = await api.getAccountInfo(address); if (!is_vesting_account_type(account['@type'])) { throw new Error( diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index 77c8bcf3..3283b415 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -15,7 +15,7 @@ export async function handler(request: Request): Promise { } let api = new NodeApi(REST_API); - const account = await api.auth_get_account(address); + const account = await api.getAccountInfo(address); if (!is_vesting_account_type(account['@type'])) { throw new Error( diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 079c7660..6a5d0149 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -11,19 +11,19 @@ export async function get_account_balance_infos_from_node_api( address: string ): Promise { const node_api = new NodeApi(REST_API); - const available_balance = await node_api.bank_get_account_balances(address); + const available_balance = await node_api.getAvailableBalance(address); let available_balance_in_ncheq = 0; if (available_balance.length > 0) { available_balance_in_ncheq = Number(available_balance[0]?.amount); } - const reward_balance_in_ncheq = await node_api.distribution_get_total_rewards( + const reward_balance_in_ncheq = await node_api.distributionGetRewards( address ); const total_delegation_balance_in_ncheq = await calculate_total_delegations_balance_for_delegator_in_ncheq( - await node_api.staking_get_all_delegations_for_delegator( + await node_api.getAllDelegations( address, 0, // first call true @@ -33,7 +33,7 @@ export async function get_account_balance_infos_from_node_api( const total_unbonding_balance_in_ncheq = await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( - await node_api.staking_get_all_unbonding_delegations_for_delegator( + await node_api.getAllUnbondingDelegations( address, 0, // first call true @@ -76,7 +76,7 @@ export async function calculate_total_delegations_balance_for_delegator_in_ncheq const delegator_address = delegationsResp.delegation_responses[0].delegation.delegator_address; - const resp = await node_api.staking_get_all_delegations_for_delegator( + const resp = await node_api.getAllDelegations( delegator_address, current_offset, // our current offset will be updated by recursive call below true // we count total again , since it's implemented recursively @@ -116,7 +116,7 @@ export async function calculate_total_unbonding_delegations_balance_for_delegato unbondingResp.unbonding_responses[0].delegator_address; const resp = - await node_api.staking_get_all_unbonding_delegations_for_delegator( + await node_api.getAllUnbondingDelegations( delegator_address, current_offset, true diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index a8a91cdc..2377c0e8 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -3,6 +3,8 @@ import { ncheq_to_cheq_fixed } from '../helpers/currency'; import { NodeApi } from '../api/nodeApi'; import { AccountBalanceInfos } from '../types/node'; import { extract_group_number_and_address } from './kv'; +import { BigDipperApi } from '../api/bigDipperApi'; +import { GraphQLClient } from '../helpers/graphql'; export async function updateCirculatingSupply(groupNumber: number) { try { @@ -57,16 +59,11 @@ export async function updateCachedBalance(addr: string, grpN: number) { } } -export async function getTotalSupply(): Promise { - let node_api = new NodeApi(REST_API); - let total_supply_ncheq = await node_api.bank_getTotalSupply_ncheq(); - const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); - - return total_supply; -} - export async function get_circulating_supply(): Promise { - const total_supply = await getTotalSupply(); + let gql_client = new GraphQLClient(GRAPHQL_API); + let bd_api = new BigDipperApi(gql_client); + let total_supply_ncheq = await bd_api.getTotalSupply(); + const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); From 02cdef6814a83f67025a898d9613a64cbc69f6c2 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 16:30:05 +0000 Subject: [PATCH 201/224] Rename circulating supply --- src/handlers/circulatingSupply.ts | 4 ++-- src/helpers/circulating.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 3fc71810..22dcecf5 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -1,8 +1,8 @@ import { Request } from 'itty-router'; -import { get_circulating_supply } from '../helpers/circulating'; +import { getCirculatingSupply } from '../helpers/circulating'; export async function handler(request: Request): Promise { - let circulating_supply = await get_circulating_supply(); + let circulating_supply = await getCirculatingSupply(); return new Response(circulating_supply.toString()); } diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 2377c0e8..02d96605 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -59,7 +59,7 @@ export async function updateCachedBalance(addr: string, grpN: number) { } } -export async function get_circulating_supply(): Promise { +export async function getCirculatingSupply(): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); let total_supply_ncheq = await bd_api.getTotalSupply(); From 46dab1f8a75260a8d9c8b59661e691b7d8988fc6 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 16:30:20 +0000 Subject: [PATCH 202/224] Delete delegatorCount.ts --- src/handlers/delegatorCount.ts | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 src/handlers/delegatorCount.ts diff --git a/src/handlers/delegatorCount.ts b/src/handlers/delegatorCount.ts deleted file mode 100644 index 4715df6e..00000000 --- a/src/handlers/delegatorCount.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Request } from 'itty-router'; -import { try_getting_delegators_count_from_KV } from '../helpers/validators'; - -export async function handler(request: Request): Promise { - const address = request.params?.['validator_address']; - - if (!address) { - throw new Error('No address specified or wrong address format.'); - } - - try { - const total_delegators_from_cache = - await try_getting_delegators_count_from_KV(address); - if (!total_delegators_from_cache) { - throw new Error('No delegators count cached for given validator'); - } - return new Response(JSON.stringify(total_delegators_from_cache)); - } catch (error) { - console.log(error); - throw new Error('Error while getting delegators count for validator'); - } -} From 9fa497afab6057bea7e0b45e8875daa6ebebbc91 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 17:23:43 +0000 Subject: [PATCH 203/224] Rename to isValidAddress --- src/handlers/liquidBalance.ts | 4 ++-- src/handlers/vestedBalance.ts | 4 ++-- src/handlers/vestingBalance.ts | 4 ++-- src/helpers/validate.ts | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index 53112083..6972fe1a 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -1,5 +1,5 @@ import { Request } from "itty-router"; -import { is_delayed_vesting_account_type, is_vesting_account_type, validate_cheqd_address } from "../helpers/validate"; +import { is_delayed_vesting_account_type, is_vesting_account_type, isValidAddress } from "../helpers/validate"; import { NodeApi } from "../api/nodeApi"; import { calculate_vested_coins } from "../helpers/vesting"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; @@ -7,7 +7,7 @@ import { ncheq_to_cheq_fixed } from "../helpers/currency"; export async function handler(request: Request): Promise { const address = request.params?.['address']; - if (!address || !validate_cheqd_address(address)) { + if (!address || !isValidAddress(address)) { throw new Error("No address specified or wrong address format."); } diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index eea79b11..46327af5 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -1,7 +1,7 @@ import { Request } from 'itty-router'; import { is_vesting_account_type, - validate_cheqd_address, + isValidAddress, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; import { estimatedVesting } from '../helpers/vesting'; @@ -10,7 +10,7 @@ import { ncheq_to_cheq_fixed } from '../helpers/currency'; export async function handler(request: Request): Promise { const address = request.params?.['address']; - if (!address || !validate_cheqd_address(address)) { + if (!address || !isValidAddress(address)) { throw new Error('No address specified or wrong address format.'); } diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index 3283b415..bb5a2497 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -1,7 +1,7 @@ import { Request } from 'itty-router'; import { is_vesting_account_type, - validate_cheqd_address, + isValidAddress, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; import { estimatedVesting } from '../helpers/vesting'; @@ -10,7 +10,7 @@ import { ncheq_to_cheq_fixed } from '../helpers/currency'; export async function handler(request: Request): Promise { const address = request.params?.['address']; - if (!address || !validate_cheqd_address(address)) { + if (!address || !isValidAddress(address)) { throw new Error('No address specified or wrong address format.'); } diff --git a/src/helpers/validate.ts b/src/helpers/validate.ts index 8b80fce8..fe207159 100644 --- a/src/helpers/validate.ts +++ b/src/helpers/validate.ts @@ -1,5 +1,5 @@ // TODO: This doesn't take checksum into account -export function validate_cheqd_address(address: string): boolean { +export function isValidAddress(address: string): boolean { return /^(cheqd)1[a-z0-9]{38}$/.test(address) } From b484db266125519516ae9d6c0e6d0106087b1058 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 17:24:51 +0000 Subject: [PATCH 204/224] Rename to isValidAddress --- src/handlers/liquidBalance.ts | 4 ++-- src/handlers/vestedBalance.ts | 4 ++-- src/handlers/vestingBalance.ts | 4 ++-- src/helpers/validate.ts | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index 6972fe1a..ab94b86b 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -1,5 +1,5 @@ import { Request } from "itty-router"; -import { is_delayed_vesting_account_type, is_vesting_account_type, isValidAddress } from "../helpers/validate"; +import { is_delayed_vesting_account_type, isVestingAccount, isValidAddress } from "../helpers/validate"; import { NodeApi } from "../api/nodeApi"; import { calculate_vested_coins } from "../helpers/vesting"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; @@ -14,7 +14,7 @@ export async function handler(request: Request): Promise { let api = new NodeApi(REST_API); const account = await api.getAccountInfo(address) - if (!is_vesting_account_type(account["@type"])) { + if (!isVestingAccount(account["@type"])) { throw new Error(`Only vesting accounts are supported. Accounts type '${account["@type"]}'.`) } diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index 46327af5..e7b0a33e 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -1,6 +1,6 @@ import { Request } from 'itty-router'; import { - is_vesting_account_type, + isVestingAccount, isValidAddress, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; @@ -17,7 +17,7 @@ export async function handler(request: Request): Promise { let api = new NodeApi(REST_API); const account = await api.getAccountInfo(address); - if (!is_vesting_account_type(account['@type'])) { + if (!isVestingAccount(account['@type'])) { throw new Error( `Only vesting accounts are supported. Accounts type '${account['@type']}'.` ); diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index bb5a2497..45bdd5d4 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -1,6 +1,6 @@ import { Request } from 'itty-router'; import { - is_vesting_account_type, + isVestingAccount, isValidAddress, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; @@ -17,7 +17,7 @@ export async function handler(request: Request): Promise { let api = new NodeApi(REST_API); const account = await api.getAccountInfo(address); - if (!is_vesting_account_type(account['@type'])) { + if (!isVestingAccount(account['@type'])) { throw new Error( `Only vesting accounts are supported. Accounts type '${account['@type']}'.` ); diff --git a/src/helpers/validate.ts b/src/helpers/validate.ts index fe207159..61af58f6 100644 --- a/src/helpers/validate.ts +++ b/src/helpers/validate.ts @@ -3,7 +3,7 @@ export function isValidAddress(address: string): boolean { return /^(cheqd)1[a-z0-9]{38}$/.test(address) } -export function is_vesting_account_type(account_type: string): boolean { +export function isVestingAccount(account_type: string): boolean { return account_type === '/cosmos.vesting.v1beta1.ContinuousVestingAccount' || account_type === '/cosmos.vesting.v1beta1.DelayedVestingAccount'; } From 0de1ac76a27e8b53a00b49dd7b496823ee728cc2 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 17:25:22 +0000 Subject: [PATCH 205/224] Rename to isDelayedVestingAccount --- src/handlers/liquidBalance.ts | 4 ++-- src/helpers/validate.ts | 2 +- src/helpers/vesting.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index ab94b86b..29d668e6 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -1,5 +1,5 @@ import { Request } from "itty-router"; -import { is_delayed_vesting_account_type, isVestingAccount, isValidAddress } from "../helpers/validate"; +import { isDelayedVestingAccount, isVestingAccount, isValidAddress } from "../helpers/validate"; import { NodeApi } from "../api/nodeApi"; import { calculate_vested_coins } from "../helpers/vesting"; import { ncheq_to_cheq_fixed } from "../helpers/currency"; @@ -18,7 +18,7 @@ export async function handler(request: Request): Promise { throw new Error(`Only vesting accounts are supported. Accounts type '${account["@type"]}'.`) } - if(is_delayed_vesting_account_type(account?.["@type"])) { + if(isDelayedVestingAccount(account?.["@type"])) { let balance = account?.base_vesting_account?.base_account?.sequence !== '0' ? Number(await (await api.getAvailableBalance(address)).find(b => b.denom === "ncheq")?.amount ?? '0') : 0; let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let delegated = Number(account?.base_vesting_account?.delegated_free?.find(d => d.denom === "ncheq")?.amount ?? '0'); diff --git a/src/helpers/validate.ts b/src/helpers/validate.ts index 61af58f6..90eee4bc 100644 --- a/src/helpers/validate.ts +++ b/src/helpers/validate.ts @@ -11,6 +11,6 @@ export function is_continuous_vesting_account_type(account_type: string): boolea return account_type === '/cosmos.vesting.v1beta1.ContinuousVestingAccount'; } -export function is_delayed_vesting_account_type(account_type: string): boolean { +export function isDelayedVestingAccount(account_type: string): boolean { return account_type === '/cosmos.vesting.v1beta1.DelayedVestingAccount'; } diff --git a/src/helpers/vesting.ts b/src/helpers/vesting.ts index 8964ba60..0d12a8b5 100644 --- a/src/helpers/vesting.ts +++ b/src/helpers/vesting.ts @@ -1,7 +1,7 @@ import { Account } from '../types/node'; import { is_continuous_vesting_account_type, - is_delayed_vesting_account_type, + isDelayedVestingAccount, } from './validate'; // TODO: This method computes the amount of coins vested. This is not the same as coins that user can spend. @@ -65,7 +65,7 @@ export function estimatedVesting(account: Account, t?: Date) { vesting, }; } - if (is_delayed_vesting_account_type(account?.['@type'])) { + if (isDelayedVestingAccount(account?.['@type'])) { const endsAt = account.base_vesting_account.end_time; const originalVesting = Number( From c998576706b7110d93193af8867c4d7859f47791 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 17:29:30 +0000 Subject: [PATCH 206/224] Rename to convertToLargestDenom --- src/handlers/liquidBalance.ts | 6 +++--- src/handlers/totalStakedCoins.ts | 4 ++-- src/handlers/totalSupply.ts | 4 ++-- src/handlers/vestedBalance.ts | 4 ++-- src/handlers/vestingBalance.ts | 4 ++-- src/helpers/balance.ts | 12 ++++++------ src/helpers/circulating.ts | 4 ++-- src/helpers/currency.ts | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index 29d668e6..84e00fe9 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -2,7 +2,7 @@ import { Request } from "itty-router"; import { isDelayedVestingAccount, isVestingAccount, isValidAddress } from "../helpers/validate"; import { NodeApi } from "../api/nodeApi"; import { calculate_vested_coins } from "../helpers/vesting"; -import { ncheq_to_cheq_fixed } from "../helpers/currency"; +import { convertToLargestDenom } from "../helpers/currency"; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -23,7 +23,7 @@ export async function handler(request: Request): Promise { let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let delegated = Number(account?.base_vesting_account?.delegated_free?.find(d => d.denom === "ncheq")?.amount ?? '0'); - return new Response(ncheq_to_cheq_fixed(balance + rewards + delegated)); + return new Response(convertToLargestDenom(balance + rewards + delegated)); } let vested_coins = calculate_vested_coins(account); @@ -31,5 +31,5 @@ export async function handler(request: Request): Promise { let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let liquid_coins = vested_coins + balance + rewards; - return new Response(ncheq_to_cheq_fixed(liquid_coins)); + return new Response(convertToLargestDenom(liquid_coins)); } diff --git a/src/handlers/totalStakedCoins.ts b/src/handlers/totalStakedCoins.ts index 25ddd23d..862c4a4d 100644 --- a/src/handlers/totalStakedCoins.ts +++ b/src/handlers/totalStakedCoins.ts @@ -1,6 +1,6 @@ import { Request } from 'itty-router'; import { BigDipperApi } from '../api/bigDipperApi'; -import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { convertToLargestDenom } from '../helpers/currency'; import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { @@ -9,5 +9,5 @@ export async function handler(request: Request): Promise { let total_staked_coins = await bd_api.getTotalStakedCoins(); - return new Response(ncheq_to_cheq_fixed(Number(total_staked_coins))); + return new Response(convertToLargestDenom(Number(total_staked_coins))); } diff --git a/src/handlers/totalSupply.ts b/src/handlers/totalSupply.ts index e5c1c975..75a402d6 100644 --- a/src/handlers/totalSupply.ts +++ b/src/handlers/totalSupply.ts @@ -1,11 +1,11 @@ import { Request } from 'itty-router'; import { BigDipperApi } from '../api/bigDipperApi'; -import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { convertToLargestDenom } from '../helpers/currency'; import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); const total_supply = await bd_api.getTotalSupply(); - return new Response(ncheq_to_cheq_fixed(total_supply)); + return new Response(convertToLargestDenom(total_supply)); } diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index e7b0a33e..305d4d7f 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -5,7 +5,7 @@ import { } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; import { estimatedVesting } from '../helpers/vesting'; -import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { convertToLargestDenom } from '../helpers/currency'; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -25,5 +25,5 @@ export async function handler(request: Request): Promise { let vested_coins = estimatedVesting(account)?.vested; - return new Response(ncheq_to_cheq_fixed(vested_coins!!)); + return new Response(convertToLargestDenom(vested_coins!!)); } diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index 45bdd5d4..a506fa01 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -5,7 +5,7 @@ import { } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; import { estimatedVesting } from '../helpers/vesting'; -import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { convertToLargestDenom } from '../helpers/currency'; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -25,5 +25,5 @@ export async function handler(request: Request): Promise { let vestingCoins = estimatedVesting(account)?.vesting; - return new Response(ncheq_to_cheq_fixed(vestingCoins!!)); + return new Response(convertToLargestDenom(vestingCoins!!)); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 6a5d0149..696734e4 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -4,7 +4,7 @@ import { DelegationsResponse, UnbondingResponse } from '../types/node'; -import { ncheq_to_cheq_fixed } from './currency'; +import { convertToLargestDenom } from './currency'; import { } from '../types/node'; export async function get_account_balance_infos_from_node_api( @@ -43,17 +43,17 @@ export async function get_account_balance_infos_from_node_api( return { totalBalance: Number( - ncheq_to_cheq_fixed( + convertToLargestDenom( available_balance_in_ncheq + reward_balance_in_ncheq + total_delegation_balance_in_ncheq + total_unbonding_balance_in_ncheq ) ), - availableBalance: Number(ncheq_to_cheq_fixed(available_balance_in_ncheq)), - rewards: Number(ncheq_to_cheq_fixed(reward_balance_in_ncheq)), - delegated: Number(ncheq_to_cheq_fixed(total_delegation_balance_in_ncheq)), - unbonding: Number(ncheq_to_cheq_fixed(total_unbonding_balance_in_ncheq)), + availableBalance: Number(convertToLargestDenom(available_balance_in_ncheq)), + rewards: Number(convertToLargestDenom(reward_balance_in_ncheq)), + delegated: Number(convertToLargestDenom(total_delegation_balance_in_ncheq)), + unbonding: Number(convertToLargestDenom(total_unbonding_balance_in_ncheq)), timeUpdated: new Date().toUTCString(), }; } diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 02d96605..ac8c1f7d 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -1,5 +1,5 @@ import { get_account_balance_infos_from_node_api } from './balance'; -import { ncheq_to_cheq_fixed } from '../helpers/currency'; +import { convertToLargestDenom } from '../helpers/currency'; import { NodeApi } from '../api/nodeApi'; import { AccountBalanceInfos } from '../types/node'; import { extract_group_number_and_address } from './kv'; @@ -63,7 +63,7 @@ export async function getCirculatingSupply(): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); let total_supply_ncheq = await bd_api.getTotalSupply(); - const total_supply = Number(ncheq_to_cheq_fixed(total_supply_ncheq)); + const total_supply = Number(convertToLargestDenom(total_supply_ncheq)); try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); diff --git a/src/helpers/currency.ts b/src/helpers/currency.ts index 82d8dadc..2add778b 100644 --- a/src/helpers/currency.ts +++ b/src/helpers/currency.ts @@ -8,6 +8,6 @@ export function cheqd_to_ncheq(cheqd: number): number { return cheqd * TOKEN_DECIMALS; } -export function ncheq_to_cheq_fixed(ncheq: number): string { +export function convertToLargestDenom(ncheq: number): string { return ncheq_to_cheqd(ncheq).toFixed(0); } \ No newline at end of file From 281a4c62bb1932bb9fca48daf3a19492f814d20c Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 17:33:58 +0000 Subject: [PATCH 207/224] Rename to convertToMainTokenDenom --- src/handlers/liquidBalance.ts | 6 +++--- src/handlers/totalStakedCoins.ts | 4 ++-- src/handlers/totalSupply.ts | 4 ++-- src/handlers/vestedBalance.ts | 4 ++-- src/handlers/vestingBalance.ts | 4 ++-- src/helpers/balance.ts | 12 ++++++------ src/helpers/circulating.ts | 4 ++-- src/helpers/currency.ts | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index 84e00fe9..72cba202 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -2,7 +2,7 @@ import { Request } from "itty-router"; import { isDelayedVestingAccount, isVestingAccount, isValidAddress } from "../helpers/validate"; import { NodeApi } from "../api/nodeApi"; import { calculate_vested_coins } from "../helpers/vesting"; -import { convertToLargestDenom } from "../helpers/currency"; +import { convertToMainTokenDenom } from "../helpers/currency"; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -23,7 +23,7 @@ export async function handler(request: Request): Promise { let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let delegated = Number(account?.base_vesting_account?.delegated_free?.find(d => d.denom === "ncheq")?.amount ?? '0'); - return new Response(convertToLargestDenom(balance + rewards + delegated)); + return new Response(convertToMainTokenDenom(balance + rewards + delegated)); } let vested_coins = calculate_vested_coins(account); @@ -31,5 +31,5 @@ export async function handler(request: Request): Promise { let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let liquid_coins = vested_coins + balance + rewards; - return new Response(convertToLargestDenom(liquid_coins)); + return new Response(convertToMainTokenDenom(liquid_coins)); } diff --git a/src/handlers/totalStakedCoins.ts b/src/handlers/totalStakedCoins.ts index 862c4a4d..cb7abf89 100644 --- a/src/handlers/totalStakedCoins.ts +++ b/src/handlers/totalStakedCoins.ts @@ -1,6 +1,6 @@ import { Request } from 'itty-router'; import { BigDipperApi } from '../api/bigDipperApi'; -import { convertToLargestDenom } from '../helpers/currency'; +import { convertToMainTokenDenom } from '../helpers/currency'; import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { @@ -9,5 +9,5 @@ export async function handler(request: Request): Promise { let total_staked_coins = await bd_api.getTotalStakedCoins(); - return new Response(convertToLargestDenom(Number(total_staked_coins))); + return new Response(convertToMainTokenDenom(Number(total_staked_coins))); } diff --git a/src/handlers/totalSupply.ts b/src/handlers/totalSupply.ts index 75a402d6..55c950b5 100644 --- a/src/handlers/totalSupply.ts +++ b/src/handlers/totalSupply.ts @@ -1,11 +1,11 @@ import { Request } from 'itty-router'; import { BigDipperApi } from '../api/bigDipperApi'; -import { convertToLargestDenom } from '../helpers/currency'; +import { convertToMainTokenDenom } from '../helpers/currency'; import { GraphQLClient } from '../helpers/graphql'; export async function handler(request: Request): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); const total_supply = await bd_api.getTotalSupply(); - return new Response(convertToLargestDenom(total_supply)); + return new Response(convertToMainTokenDenom(total_supply)); } diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index 305d4d7f..9b9db6a1 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -5,7 +5,7 @@ import { } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; import { estimatedVesting } from '../helpers/vesting'; -import { convertToLargestDenom } from '../helpers/currency'; +import { convertToMainTokenDenom } from '../helpers/currency'; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -25,5 +25,5 @@ export async function handler(request: Request): Promise { let vested_coins = estimatedVesting(account)?.vested; - return new Response(convertToLargestDenom(vested_coins!!)); + return new Response(convertToMainTokenDenom(vested_coins!!)); } diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index a506fa01..2a64e51f 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -5,7 +5,7 @@ import { } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; import { estimatedVesting } from '../helpers/vesting'; -import { convertToLargestDenom } from '../helpers/currency'; +import { convertToMainTokenDenom } from '../helpers/currency'; export async function handler(request: Request): Promise { const address = request.params?.['address']; @@ -25,5 +25,5 @@ export async function handler(request: Request): Promise { let vestingCoins = estimatedVesting(account)?.vesting; - return new Response(convertToLargestDenom(vestingCoins!!)); + return new Response(convertToMainTokenDenom(vestingCoins!!)); } diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 696734e4..75e1b619 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -4,7 +4,7 @@ import { DelegationsResponse, UnbondingResponse } from '../types/node'; -import { convertToLargestDenom } from './currency'; +import { convertToMainTokenDenom } from './currency'; import { } from '../types/node'; export async function get_account_balance_infos_from_node_api( @@ -43,17 +43,17 @@ export async function get_account_balance_infos_from_node_api( return { totalBalance: Number( - convertToLargestDenom( + convertToMainTokenDenom( available_balance_in_ncheq + reward_balance_in_ncheq + total_delegation_balance_in_ncheq + total_unbonding_balance_in_ncheq ) ), - availableBalance: Number(convertToLargestDenom(available_balance_in_ncheq)), - rewards: Number(convertToLargestDenom(reward_balance_in_ncheq)), - delegated: Number(convertToLargestDenom(total_delegation_balance_in_ncheq)), - unbonding: Number(convertToLargestDenom(total_unbonding_balance_in_ncheq)), + availableBalance: Number(convertToMainTokenDenom(available_balance_in_ncheq)), + rewards: Number(convertToMainTokenDenom(reward_balance_in_ncheq)), + delegated: Number(convertToMainTokenDenom(total_delegation_balance_in_ncheq)), + unbonding: Number(convertToMainTokenDenom(total_unbonding_balance_in_ncheq)), timeUpdated: new Date().toUTCString(), }; } diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index ac8c1f7d..6e83b933 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -1,5 +1,5 @@ import { get_account_balance_infos_from_node_api } from './balance'; -import { convertToLargestDenom } from '../helpers/currency'; +import { convertToMainTokenDenom } from '../helpers/currency'; import { NodeApi } from '../api/nodeApi'; import { AccountBalanceInfos } from '../types/node'; import { extract_group_number_and_address } from './kv'; @@ -63,7 +63,7 @@ export async function getCirculatingSupply(): Promise { let gql_client = new GraphQLClient(GRAPHQL_API); let bd_api = new BigDipperApi(gql_client); let total_supply_ncheq = await bd_api.getTotalSupply(); - const total_supply = Number(convertToLargestDenom(total_supply_ncheq)); + const total_supply = Number(convertToMainTokenDenom(total_supply_ncheq)); try { const cached = await CIRCULATING_SUPPLY_WATCHLIST.list(); diff --git a/src/helpers/currency.ts b/src/helpers/currency.ts index 2add778b..9768df9d 100644 --- a/src/helpers/currency.ts +++ b/src/helpers/currency.ts @@ -8,6 +8,6 @@ export function cheqd_to_ncheq(cheqd: number): number { return cheqd * TOKEN_DECIMALS; } -export function convertToLargestDenom(ncheq: number): string { +export function convertToMainTokenDenom(ncheq: number): string { return ncheq_to_cheqd(ncheq).toFixed(0); } \ No newline at end of file From 30349eb1f048a22dd779637e8b8858355bd02e19 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 18:12:19 +0000 Subject: [PATCH 208/224] Rename to fetchAccountBalance --- src/handlers/totalBalance.ts | 4 ++-- src/helpers/balance.ts | 8 ++++---- src/helpers/circulating.ts | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/handlers/totalBalance.ts b/src/handlers/totalBalance.ts index b8c780ef..28fd0892 100644 --- a/src/handlers/totalBalance.ts +++ b/src/handlers/totalBalance.ts @@ -1,9 +1,9 @@ import { Request } from 'itty-router'; -import { get_account_balance_infos_from_node_api } from '../helpers/balance'; +import { fetchAccountBalances } from '../helpers/balance'; export async function handler(request: Request): Promise { const address = request.params?.['address']; - let account_balance_infos = await get_account_balance_infos_from_node_api( + let account_balance_infos = await fetchAccountBalances( address!! ); return new Response(account_balance_infos?.totalBalance.toString()); diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 75e1b619..0e5c02d8 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -7,7 +7,7 @@ import { import { convertToMainTokenDenom } from './currency'; import { } from '../types/node'; -export async function get_account_balance_infos_from_node_api( +export async function fetchAccountBalances( address: string ): Promise { const node_api = new NodeApi(REST_API); @@ -22,7 +22,7 @@ export async function get_account_balance_infos_from_node_api( address ); const total_delegation_balance_in_ncheq = - await calculate_total_delegations_balance_for_delegator_in_ncheq( + await calculateTotalDelegationBalance( await node_api.getAllDelegations( address, 0, // first call @@ -58,7 +58,7 @@ export async function get_account_balance_infos_from_node_api( }; } -export async function calculate_total_delegations_balance_for_delegator_in_ncheq( +export async function calculateTotalDelegationBalance( delegationsResp: DelegationsResponse, current_offset: number ): Promise { @@ -83,7 +83,7 @@ export async function calculate_total_delegations_balance_for_delegator_in_ncheq ); total_delegation_balance_in_ncheq += - await calculate_total_delegations_balance_for_delegator_in_ncheq( + await calculateTotalDelegationBalance( resp, current_offset + Number(REST_API_PAGINATION_LIMIT) ); diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index 6e83b933..a28cb3ab 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -1,4 +1,4 @@ -import { get_account_balance_infos_from_node_api } from './balance'; +import { fetchAccountBalances } from './balance'; import { convertToMainTokenDenom } from '../helpers/currency'; import { NodeApi } from '../api/nodeApi'; import { AccountBalanceInfos } from '../types/node'; @@ -45,7 +45,7 @@ export async function updateCirculatingSupply(groupNumber: number) { export async function updateCachedBalance(addr: string, grpN: number) { try { - const account_balance_infos = await get_account_balance_infos_from_node_api( + const account_balance_infos = await fetchAccountBalances( addr ); From d2e1269a71c782dda01472b0d51eab3036cf150f Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 18:12:52 +0000 Subject: [PATCH 209/224] Rename to calculateTotalUnbondingBalance --- src/helpers/balance.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/helpers/balance.ts b/src/helpers/balance.ts index 0e5c02d8..656549c7 100644 --- a/src/helpers/balance.ts +++ b/src/helpers/balance.ts @@ -32,7 +32,7 @@ export async function fetchAccountBalances( ); const total_unbonding_balance_in_ncheq = - await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( + await calculateTotalUnbondingBalance( await node_api.getAllUnbondingDelegations( address, 0, // first call @@ -92,7 +92,7 @@ export async function calculateTotalDelegationBalance( return total_delegation_balance_in_ncheq; } -export async function calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( +export async function calculateTotalUnbondingBalance( unbondingResp: UnbondingResponse, current_offset: number ): Promise { @@ -123,7 +123,7 @@ export async function calculate_total_unbonding_delegations_balance_for_delegato ); total_unbonding_balance_in_ncheq += - await calculate_total_unbonding_delegations_balance_for_delegator_in_ncheq( + await calculateTotalUnbondingBalance( resp, current_offset + Number(REST_API_PAGINATION_LIMIT) ); From cbe77dc789a1096a8ad803177d1c5e4af6dfcfdc Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 18:43:26 +0000 Subject: [PATCH 210/224] Rename to extractPrefixAndKey --- src/helpers/circulating.ts | 5 ++--- src/helpers/kv.ts | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/helpers/circulating.ts b/src/helpers/circulating.ts index a28cb3ab..b4b9e850 100644 --- a/src/helpers/circulating.ts +++ b/src/helpers/circulating.ts @@ -1,8 +1,7 @@ import { fetchAccountBalances } from './balance'; import { convertToMainTokenDenom } from '../helpers/currency'; -import { NodeApi } from '../api/nodeApi'; import { AccountBalanceInfos } from '../types/node'; -import { extract_group_number_and_address } from './kv'; +import { extractPrefixAndKey } from './kv'; import { BigDipperApi } from '../api/bigDipperApi'; import { GraphQLClient } from '../helpers/graphql'; @@ -17,7 +16,7 @@ export async function updateCirculatingSupply(groupNumber: number) { ); for (const key of cached.keys) { - const parts = extract_group_number_and_address(key.name); + const parts = extractPrefixAndKey(key.name); let addr = parts.address; let grpN = parts.groupNumber; diff --git a/src/helpers/kv.ts b/src/helpers/kv.ts index bedde805..722c4205 100644 --- a/src/helpers/kv.ts +++ b/src/helpers/kv.ts @@ -1,4 +1,4 @@ -export function extract_group_number_and_address(key: string) { +export function extractPrefixAndKey(key: string) { const parts = key.split(':'); let addr = parts[1]; let grpN = Number(parts[0].split('_')[1]); From a2e62d5c99f33e0a89f4606565d19290babd698c Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 18:49:16 +0000 Subject: [PATCH 211/224] Delete unused function --- src/helpers/currency.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/helpers/currency.ts b/src/helpers/currency.ts index 9768df9d..560224e1 100644 --- a/src/helpers/currency.ts +++ b/src/helpers/currency.ts @@ -1,13 +1,9 @@ import { TOKEN_DECIMALS } from "./constants"; -export function ncheq_to_cheqd(ncheq: number): number { +export function convertToLowestDenom(ncheq: number): number { return ncheq / TOKEN_DECIMALS; } -export function cheqd_to_ncheq(cheqd: number): number { - return cheqd * TOKEN_DECIMALS; -} - export function convertToMainTokenDenom(ncheq: number): string { - return ncheq_to_cheqd(ncheq).toFixed(0); -} \ No newline at end of file + return convertToLowestDenom(ncheq).toFixed(0); +} From ef9371eaea8d98fb42a3c1d9de921817e8212507 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 18:50:31 +0000 Subject: [PATCH 212/224] Rename to isContinuousVestingAccount --- src/helpers/validate.ts | 2 +- src/helpers/vesting.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/helpers/validate.ts b/src/helpers/validate.ts index 90eee4bc..aff5344a 100644 --- a/src/helpers/validate.ts +++ b/src/helpers/validate.ts @@ -7,7 +7,7 @@ export function isVestingAccount(account_type: string): boolean { return account_type === '/cosmos.vesting.v1beta1.ContinuousVestingAccount' || account_type === '/cosmos.vesting.v1beta1.DelayedVestingAccount'; } -export function is_continuous_vesting_account_type(account_type: string): boolean { +export function isContinuousVestingAccount(account_type: string): boolean { return account_type === '/cosmos.vesting.v1beta1.ContinuousVestingAccount'; } diff --git a/src/helpers/vesting.ts b/src/helpers/vesting.ts index 0d12a8b5..d5a10e74 100644 --- a/src/helpers/vesting.ts +++ b/src/helpers/vesting.ts @@ -1,6 +1,6 @@ import { Account } from '../types/node'; import { - is_continuous_vesting_account_type, + isContinuousVestingAccount, isDelayedVestingAccount, } from './validate'; @@ -44,7 +44,7 @@ export function estimatedVesting(account: Account, t?: Date) { t = new Date(); } - if (is_continuous_vesting_account_type(account?.['@type'])) { + if (isContinuousVestingAccount(account?.['@type'])) { const startsAt = account.start_time; const endsAt = account.base_vesting_account.end_time; From 6afcb59e4723fb399c13280e6864090d360d944a Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 18:53:28 +0000 Subject: [PATCH 213/224] Rename to calculateVestedBalance --- src/handlers/liquidBalance.ts | 4 ++-- src/helpers/vesting.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index 72cba202..71600809 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -1,7 +1,7 @@ import { Request } from "itty-router"; import { isDelayedVestingAccount, isVestingAccount, isValidAddress } from "../helpers/validate"; import { NodeApi } from "../api/nodeApi"; -import { calculate_vested_coins } from "../helpers/vesting"; +import { calculateVestedBalance } from "../helpers/vesting"; import { convertToMainTokenDenom } from "../helpers/currency"; export async function handler(request: Request): Promise { @@ -26,7 +26,7 @@ export async function handler(request: Request): Promise { return new Response(convertToMainTokenDenom(balance + rewards + delegated)); } - let vested_coins = calculate_vested_coins(account); + let vested_coins = calculateVestedBalance(account); let balance = Number(await (await api.getAvailableBalance(address)).find(b => b.denom === "ncheq")?.amount ?? '0') let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let liquid_coins = vested_coins + balance + rewards; diff --git a/src/helpers/vesting.ts b/src/helpers/vesting.ts index d5a10e74..954ce1aa 100644 --- a/src/helpers/vesting.ts +++ b/src/helpers/vesting.ts @@ -8,7 +8,7 @@ import { // To calculate spendable tokens we need to take into account initial balance + sent and received tokens as well. // Here is the explanation of how to do it properly: // https://docs.cosmos.network/master/modules/auth/05_vesting.html#transferring-sending -export function calculate_vested_coins(account: Account): number { +export function calculateVestedBalance(account: Account): number { if ( account?.['@type'] === '/cosmos.vesting.v1beta1.DelayedVestingAccount' && Date.now() < account?.base_vesting_account?.end_time * 1000 @@ -34,7 +34,7 @@ export function calculate_vested_coins(account: Account): number { export function calculate_vesting_coins(account: Account): number { return ( Number(account.base_vesting_account.original_vesting[0].amount) - - calculate_vested_coins(account) + calculateVestedBalance(account) ); } From 1c24f18055fc5bb464507d25cea3dde0bb77b98f Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:09:42 +0000 Subject: [PATCH 214/224] Simplify vesting account calculation --- src/handlers/liquidBalance.ts | 4 ++-- src/handlers/vestedBalance.ts | 4 ++-- src/handlers/vestingBalance.ts | 4 ++-- src/helpers/vesting.ts | 36 +--------------------------------- 4 files changed, 7 insertions(+), 41 deletions(-) diff --git a/src/handlers/liquidBalance.ts b/src/handlers/liquidBalance.ts index 71600809..eb34969b 100644 --- a/src/handlers/liquidBalance.ts +++ b/src/handlers/liquidBalance.ts @@ -1,7 +1,7 @@ import { Request } from "itty-router"; import { isDelayedVestingAccount, isVestingAccount, isValidAddress } from "../helpers/validate"; import { NodeApi } from "../api/nodeApi"; -import { calculateVestedBalance } from "../helpers/vesting"; +import { calculateVesting } from "../helpers/vesting"; import { convertToMainTokenDenom } from "../helpers/currency"; export async function handler(request: Request): Promise { @@ -26,7 +26,7 @@ export async function handler(request: Request): Promise { return new Response(convertToMainTokenDenom(balance + rewards + delegated)); } - let vested_coins = calculateVestedBalance(account); + let vested_coins = Number(calculateVesting(account)?.vested); let balance = Number(await (await api.getAvailableBalance(address)).find(b => b.denom === "ncheq")?.amount ?? '0') let rewards = Number(await (await api.distributionGetRewards(address)) ?? '0'); let liquid_coins = vested_coins + balance + rewards; diff --git a/src/handlers/vestedBalance.ts b/src/handlers/vestedBalance.ts index 9b9db6a1..afb261fa 100644 --- a/src/handlers/vestedBalance.ts +++ b/src/handlers/vestedBalance.ts @@ -4,7 +4,7 @@ import { isValidAddress, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; -import { estimatedVesting } from '../helpers/vesting'; +import { calculateVesting } from '../helpers/vesting'; import { convertToMainTokenDenom } from '../helpers/currency'; export async function handler(request: Request): Promise { @@ -23,7 +23,7 @@ export async function handler(request: Request): Promise { ); } - let vested_coins = estimatedVesting(account)?.vested; + let vested_coins = calculateVesting(account)?.vested; return new Response(convertToMainTokenDenom(vested_coins!!)); } diff --git a/src/handlers/vestingBalance.ts b/src/handlers/vestingBalance.ts index 2a64e51f..cc34551e 100644 --- a/src/handlers/vestingBalance.ts +++ b/src/handlers/vestingBalance.ts @@ -4,7 +4,7 @@ import { isValidAddress, } from '../helpers/validate'; import { NodeApi } from '../api/nodeApi'; -import { estimatedVesting } from '../helpers/vesting'; +import { calculateVesting } from '../helpers/vesting'; import { convertToMainTokenDenom } from '../helpers/currency'; export async function handler(request: Request): Promise { @@ -23,7 +23,7 @@ export async function handler(request: Request): Promise { ); } - let vestingCoins = estimatedVesting(account)?.vesting; + let vestingCoins = calculateVesting(account)?.vesting; return new Response(convertToMainTokenDenom(vestingCoins!!)); } diff --git a/src/helpers/vesting.ts b/src/helpers/vesting.ts index 954ce1aa..a929ade5 100644 --- a/src/helpers/vesting.ts +++ b/src/helpers/vesting.ts @@ -4,42 +4,8 @@ import { isDelayedVestingAccount, } from './validate'; -// TODO: This method computes the amount of coins vested. This is not the same as coins that user can spend. -// To calculate spendable tokens we need to take into account initial balance + sent and received tokens as well. -// Here is the explanation of how to do it properly: -// https://docs.cosmos.network/master/modules/auth/05_vesting.html#transferring-sending -export function calculateVestedBalance(account: Account): number { - if ( - account?.['@type'] === '/cosmos.vesting.v1beta1.DelayedVestingAccount' && - Date.now() < account?.base_vesting_account?.end_time * 1000 - ) - return 0; - - const start_time = new Date(account.start_time * 1000).getTime(); - const end_time = new Date( - account.base_vesting_account.end_time * 1000 - ).getTime(); - const now = new Date().getTime(); - - const time_elapsed = Math.abs(now - start_time) / 1000; - const time_vested = Math.abs(end_time - start_time) / 1000; - - const ratio = Number(time_elapsed / time_vested); - - return ( - ratio * Number(account.base_vesting_account.original_vesting[0].amount) - ); -} - -export function calculate_vesting_coins(account: Account): number { - return ( - Number(account.base_vesting_account.original_vesting[0].amount) - - calculateVestedBalance(account) - ); -} - // Taken from our wallet app -export function estimatedVesting(account: Account, t?: Date) { +export function calculateVesting(account: Account, t?: Date) { if (!t) { t = new Date(); } From 9f398f6166f3a65185b5e61583b6970c8f110942 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:10:44 +0000 Subject: [PATCH 215/224] Update graphql.ts --- src/helpers/graphql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index e16a5d4c..bb9daab3 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -16,7 +16,7 @@ export class GraphQLClient { let json: { errors: any } = await resp.json() if (json.errors) { - throw new Error(`query failed: ${JSON.stringify(json.errors)}`) + throw new Error(`Query failed: ${JSON.stringify(json.errors)}`) } return json as T; From cf0e44c5bf7420ac6f9d105d9c5d73623b253caf Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:15:54 +0000 Subject: [PATCH 216/224] Update node.ts --- src/types/node.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/types/node.ts b/src/types/node.ts index a8f3c2d5..d8f4287d 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -32,16 +32,6 @@ export class Coin { } } -export class Delegation { - public amount: Coin; - public delegatorAddress: string; - - constructor(amount: Coin, delegatorAddress: string) { - this.delegatorAddress = delegatorAddress; - this.amount = amount; - } -} - export interface ValidatorDetailResponse { delegation_responses: [ { From d30d55e14567cc073ddec1a422bb062902f02c99 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:16:36 +0000 Subject: [PATCH 217/224] Change cron triggers --- wrangler.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index cca290ba..05d0469a 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -57,7 +57,7 @@ MARKET_MONITORING_API = "https://market-monitoring.cheqd.net" # Run `echo | wrangler secret put ` for each of these [triggers] -crons = ["0/10 * * * *"] +crons = ["0/2 * * * *"] ############################################################### ### SECTION 3: Local Development ### @@ -106,7 +106,7 @@ kv_namespaces = [ # Cron triggers for staging worker [env.staging.triggers] -crons = [ "0/5 * * * *"] +crons = [ "0 * * * *"] ############################################################### ### OPTIONAL: Build Configuration ### From 51c534c55353ee29a15e5cbf2d0f6ca13fd208b0 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:18:55 +0000 Subject: [PATCH 218/224] Reactivate markdown link check --- .github/workflows/lint.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 53e959bf..06d0e72b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,18 +8,18 @@ defaults: jobs: - # md-link-check: - # name: "Broken Markdown links" - # runs-on: ubuntu-latest + md-link-check: + name: "Broken Markdown links" + runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 + steps: + - uses: actions/checkout@v3 - # - name: Run Markdown link check - # uses: gaurav-nelson/github-action-markdown-link-check@v1 - # with: - # config-file: '.github/linters/mlc_config.json' - # use-quiet-mode: 'yes' + - name: Run Markdown link check + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + config-file: '.github/linters/mlc_config.json' + use-quiet-mode: 'yes' super-lint: name: "Super Linter" @@ -45,8 +45,6 @@ jobs: VALIDATE_JAVASCRIPT_ES: true VALIDATE_JSONC: true VALIDATE_MARKDOWN: true - VALIDATE_OPENAPI: true VALIDATE_TSX: true VALIDATE_TYPESCRIPT_ES: true - VALIDATE_XML: true VALIDATE_YAML: true From 95a654d9ec96b00d4a24d04388dfe71af9242aaf Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:19:22 +0000 Subject: [PATCH 219/224] Reactivate linter --- .github/workflows/dispatch.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index cd91f4d6..3c08a54d 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -6,12 +6,12 @@ concurrency: jobs: - # call-lint: - # name: "Lint" - # uses: ./.github/workflows/lint.yml + call-lint: + name: "Lint" + uses: ./.github/workflows/lint.yml call-deploy: name: "Deploy" - # needs: call-lint + needs: call-lint uses: ./.github/workflows/deploy.yml secrets: inherit From 83a332d5cf15c12ed74e5d043de6e6cb260f2e9b Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:20:02 +0000 Subject: [PATCH 220/224] Update pull-request.yml --- .github/workflows/pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index e77066f7..84abd594 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -30,4 +30,4 @@ jobs: # to override config-conventional rules, specify a relative path to your rules module, actions/checkout is required for this setting! commitlintRulesPath: "./.github/linters/.commitlint.rules.js" # default: undefined # if the PR contains a single commit, fail if the commit message and the PR title do not match - commitTitleMatch: "false" # default: 'true' + commitTitleMatch: false # default: 'true' From b6fb54f318c2aabc707e9d12f98dea3c98d3ea11 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:25:15 +0000 Subject: [PATCH 221/224] Fix broken links --- README.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index de26f85e..7b9c32c5 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ## ℹī¸ Overview -Cosmos SDK offers [APIs for built-in modules using gRPC, REST, and Tendermint RPC](https://docs.cosmos.network/master/core/grpc_rest.html). This project aims to provide simple REST APIs for data that default Cosmos SDK APIs can't provide. +Cosmos SDK offers [APIs for built-in modules using gRPC, REST, and Tendermint RPC](https://docs.cosmos.network/main/core/grpc_rest.html). This project aims to provide simple REST APIs for data that default Cosmos SDK APIs can't provide. This collection of custom APIs can be deployed as a [Cloudflare Worker](https://workers.cloudflare.com/) or compatible serverless platforms. @@ -60,20 +60,6 @@ Overall tokens staked, in CHEQ. Provides the overall amount staked pulled from the block explorer. -### đŸ—ŗ Delegator count by validator - -#### Endpoint - -[`data-api.cheqd.io/staking/delegators/`](https://data-api.cheqd.io/staking/delegators/cheqdvaloper1lg0vwuu888hu4arnt9egtqrm2662kcrtf2unrs) - -#### Response - -Number of delegators who delegate to a specific validator. - -#### Rationale - -Running a query to fetch list of delegators per validator can be a computationally-expensive query. This API periodically caches the response. - ### 🔐 Vesting Account Balance #### Endpoint @@ -190,7 +176,7 @@ The application expects these environment variables to be set on Cloudflare: 1. `TOKEN_EXPONENT`: Denominator for token (default `9` for CHEQ token). 2. `REST_API`: REST API for a Cosmos/cheqd node to target for queries. 3. `REST_API_PAGINATION_LIMIT`: Number of results to fetch in a single query, for queries that require iterating multiple times. (E.g., many account balance queries require this, to be able to get all delegations etc.) -4. `GRAPHQL_API`: GraphQL API for a BigDipper explorer instance for some queries. E.g., the GraphQL API for [cheqd's block explorer](https://explorer.cheqd.io/) is [`https://explorer-gql.cheqd.io/v1/graphql`](https://explorer-gql.cheqd.io/v1/graphql). +4. `GRAPHQL_API`: GraphQL API for a BigDipper explorer instance for some queries. E.g., the GraphQL API for [cheqd's block explorer](https://explorer.cheqd.io/) is `https://explorer-gql.cheqd.io/v1/graphql`. 5. `CIRCULATING_SUPPLY_GROUPS`: Number of sub-groups the circulating supply watchlist is split into (see sample JSON file below). This is to ensure that any lookups from APIs can be spaced out. 6. `MARKET_MONITORING_API`: Upstream API for running queries from CoinGecko API (see the [market-monitoring repository](https://github.com/cheqd/market-monitoring)). 7. `WEBHOOK_URL`: Zapier webhook URL to send market monitoring data to. Since this is a secret, it's not set in plaintext in `wrangler.toml` but passed via GitHub Actions secrets. From b6b9a9e3781ca7a1fc0529486e3bbd7d418d17ed Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:25:22 +0000 Subject: [PATCH 222/224] Bump to Node 18 --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 503af3ab..fecaa0dd 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 cache: 'npm' cache-dependency-path: '**/package-lock.json' @@ -54,7 +54,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 cache: 'npm' cache-dependency-path: '**/package-lock.json' From 1d909bd1be878610aad0ff6345129986ffb55db4 Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:29:46 +0000 Subject: [PATCH 223/224] Removed unused handler --- src/bindings.d.ts | 1 - src/index.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/bindings.d.ts b/src/bindings.d.ts index f17de7e5..931a0958 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -4,7 +4,6 @@ declare global { const REST_API_PAGINATION_LIMIT: number; const GRAPHQL_API: string; const CIRCULATING_SUPPLY_WATCHLIST: KVNamespace; - const ACTIVE_VALIDATORS: KVNamespace; const CIRCULATING_SUPPLY_GROUPS: number; const MARKET_MONITORING_API: string; const WEBHOOK_URL: string; diff --git a/src/index.ts b/src/index.ts index 2ad35c05..13d117ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,6 @@ import { handler as circulatingSupplyHandler } from './handlers/circulatingSuppl import { handler as liquidBalanceHandler } from './handlers/liquidBalance'; import { handler as vestingBalanceHandler } from './handlers/vestingBalance'; import { handler as vestedBalanceHandler } from './handlers/vestedBalance'; -import { handler as delegatorCountHandler } from './handlers/delegatorCount'; import { handler as totalStakedCoinsHandler } from './handlers/totalStakedCoins'; import { handler as allArbitrageOpportunitiesHandler } from './handlers/allArbitrageOpportunities'; import { handler as arbitrageOpportunitiesHandler } from './handlers/arbitrageOpportunities'; @@ -29,7 +28,6 @@ function registerRoutes(router: Router) { router.get('/balances/total/:address', totalBalanceHandler); router.get('/balances/vested/:address', vestedBalanceHandler); router.get('/balances/vesting/:address', vestingBalanceHandler); - router.get('/staking/delegators/:validator_address', delegatorCountHandler); router.get('/supply/circulating', circulatingSupplyHandler); router.get('/supply/staked', totalStakedCoinsHandler); router.get('/supply/total', totalSupplyHandler); From b3e5e5d2720950e6cb9a5577ef7dd80468714bdd Mon Sep 17 00:00:00 2001 From: Ankur Banerjee Date: Fri, 4 Nov 2022 19:42:12 +0000 Subject: [PATCH 224/224] Throw error on circulating supply --- src/handlers/circulatingSupply.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/handlers/circulatingSupply.ts b/src/handlers/circulatingSupply.ts index 22dcecf5..8ee599e4 100644 --- a/src/handlers/circulatingSupply.ts +++ b/src/handlers/circulatingSupply.ts @@ -2,7 +2,12 @@ import { Request } from 'itty-router'; import { getCirculatingSupply } from '../helpers/circulating'; export async function handler(request: Request): Promise { - let circulating_supply = await getCirculatingSupply(); - - return new Response(circulating_supply.toString()); + try { + let circulating_supply = await getCirculatingSupply(); + return new Response(circulating_supply.toString()); + } + catch (err: any) { + console.log(err); + throw new Error(err.message); + } }