diff --git a/README.md b/README.md index d096a1d3..5652a5dc 100644 --- a/README.md +++ b/README.md @@ -136,3 +136,10 @@ Squid tools assume a certain [project layout](https://docs.subsquid.io/basics/sq - Run `sqd deploy .` - Make branch for new version (eg v9) and push to origin - Switch back to main branch + +## Reset cloud dev version (v999) + +Useful if you made a schema change or need to reload data. + +- Check `squid.yaml` to make sure you're on v999 +- `sqd deploy . --update --hard-reset` diff --git a/db/migrations/1722542269716-Data.js b/db/migrations/1723560171572-Data.js similarity index 98% rename from db/migrations/1722542269716-Data.js rename to db/migrations/1723560171572-Data.js index 0c5b2b9e..04d13bed 100644 --- a/db/migrations/1722542269716-Data.js +++ b/db/migrations/1723560171572-Data.js @@ -1,5 +1,5 @@ -module.exports = class Data1722542269716 { - name = 'Data1722542269716' +module.exports = class Data1723560171572 { + name = 'Data1723560171572' async up(db) { await db.query(`CREATE TABLE "es_token" ("id" character varying NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "block_number" integer NOT NULL, "circulating" numeric NOT NULL, "staked" numeric NOT NULL, "total" numeric NOT NULL, CONSTRAINT "PK_69bef9eb94d9a5d42d726d1e661" PRIMARY KEY ("id"))`) @@ -223,6 +223,9 @@ module.exports = class Data1722542269716 { await db.query(`CREATE INDEX "IDX_22ebb6be552c90b2c99b165cde" ON "oeth_withdrawal_request" ("timestamp") `) await db.query(`CREATE INDEX "IDX_fd1acb337d03a48f775c22edcd" ON "oeth_withdrawal_request" ("block_number") `) await db.query(`CREATE INDEX "IDX_2733a93f9fc9cf900ac1de1a0d" ON "oeth_withdrawal_request" ("tx_hash") `) + await db.query(`CREATE TABLE "ogn_daily_stat" ("id" character varying NOT NULL, "block_number" integer NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "total_supply" numeric NOT NULL, "total_supply_usd" numeric NOT NULL, "total_staked" numeric NOT NULL, "trading_volume_usd" numeric NOT NULL, "market_cap_usd" numeric NOT NULL, "price_usd" numeric NOT NULL, "holders_over_threshold" integer NOT NULL, CONSTRAINT "PK_c87054f4663051254b7b2afa536" PRIMARY KEY ("id"))`) + await db.query(`CREATE INDEX "IDX_cb4297ef85375ee13a3446b240" ON "ogn_daily_stat" ("block_number") `) + await db.query(`CREATE INDEX "IDX_b96b9849e1e479d743ffb547c9" ON "ogn_daily_stat" ("timestamp") `) await db.query(`CREATE TABLE "ogv" ("id" character varying NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "block_number" integer NOT NULL, "circulating" numeric NOT NULL, "staked" numeric NOT NULL, "total" numeric NOT NULL, CONSTRAINT "PK_f16038abf451ce82bd03ea54ee7" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_2418a8b8b92b2f5977be761cf9" ON "ogv" ("timestamp") `) await db.query(`CREATE INDEX "IDX_b8f20bcf48e4aa77e0f48d77db" ON "ogv" ("block_number") `) @@ -579,6 +582,9 @@ module.exports = class Data1722542269716 { await db.query(`DROP INDEX "public"."IDX_22ebb6be552c90b2c99b165cde"`) await db.query(`DROP INDEX "public"."IDX_fd1acb337d03a48f775c22edcd"`) await db.query(`DROP INDEX "public"."IDX_2733a93f9fc9cf900ac1de1a0d"`) + await db.query(`DROP TABLE "ogn_daily_stat"`) + await db.query(`DROP INDEX "public"."IDX_cb4297ef85375ee13a3446b240"`) + await db.query(`DROP INDEX "public"."IDX_b96b9849e1e479d743ffb547c9"`) await db.query(`DROP TABLE "ogv"`) await db.query(`DROP INDEX "public"."IDX_2418a8b8b92b2f5977be761cf9"`) await db.query(`DROP INDEX "public"."IDX_b8f20bcf48e4aa77e0f48d77db"`) diff --git a/schema.graphql b/schema.graphql index 5160ae0b..f09ecb9d 100644 --- a/schema.graphql +++ b/schema.graphql @@ -754,6 +754,19 @@ type OETHWithdrawalRequest @entity { claimed: Boolean! txHash: String! @index } +type OGNDailyStat @entity { + id: ID! + blockNumber: Int! @index + timestamp: DateTime! @index + + totalSupply: BigInt! + totalSupplyUSD: Float! + totalStaked: BigInt! + tradingVolumeUSD: Float! + marketCapUSD: Float! + priceUSD: Float! + holdersOverThreshold: Int! +} type OGV @entity { id: ID! timestamp: DateTime! @index diff --git a/schema/ogn.graphql b/schema/ogn.graphql new file mode 100644 index 00000000..8f9f74f8 --- /dev/null +++ b/schema/ogn.graphql @@ -0,0 +1,13 @@ +type OGNDailyStat @entity { + id: ID! + blockNumber: Int! @index + timestamp: DateTime! @index + + totalSupply: BigInt! + totalSupplyUSD: Float! + totalStaked: BigInt! + tradingVolumeUSD: Float! + marketCapUSD: Float! + priceUSD: Float! + holdersOverThreshold: Int! +} diff --git a/src/main-mainnet.ts b/src/main-mainnet.ts index 077ab938..f7cfe960 100644 --- a/src/main-mainnet.ts +++ b/src/main-mainnet.ts @@ -14,9 +14,10 @@ import { XOGN_ADDRESS, } from '@utils/addresses' +import * as dailyStats from './mainnet/post-processors/daily-stats' import * as curve from './mainnet/processors/curve' -import * as legacyStaking from './mainnet/processors/legacy-staking' import { erc20s } from './mainnet/processors/erc20s' +import * as legacyStaking from './mainnet/processors/legacy-staking' import * as nativeStaking from './mainnet/processors/native-staking' import * as validate from './mainnet/validators/validate-mainnet' @@ -38,7 +39,7 @@ export const processor = { }), createFRRSProcessor({ from: 19917521, address: OGN_REWARDS_SOURCE_ADDRESS }), ], - postProcessors: [exchangeRates, processStatus('mainnet')], + postProcessors: [exchangeRates, dailyStats, processStatus('mainnet')], validators: [validate], } export default processor diff --git a/src/mainnet/post-processors/daily-stats.ts b/src/mainnet/post-processors/daily-stats.ts new file mode 100644 index 00000000..27a50491 --- /dev/null +++ b/src/mainnet/post-processors/daily-stats.ts @@ -0,0 +1,110 @@ +import dayjs from 'dayjs' +import utc from 'dayjs/plugin/utc' +import { FindOptionsOrderValue, LessThanOrEqual, MoreThanOrEqual } from 'typeorm' + +import { ERC20Balance, ERC20Holder, ERC20State, OGNDailyStat } from '@model' +import { Context } from '@processor' +import { EvmBatchProcessor } from '@subsquid/evm-processor' +import { applyCoingeckoData } from '@utils/coingecko' + +dayjs.extend(utc) + +export const from = 15491391 + +export const setup = async (processor: EvmBatchProcessor) => { + processor.includeAllBlocks({ from }) +} + +export const process = async (ctx: Context) => { + const firstBlockTimestamp = ctx.blocks.find((b) => b.header.height >= from)?.header.timestamp + if (!firstBlockTimestamp) return + + const firstBlock = ctx.blocks[0] + const lastBlock = ctx.blocks[ctx.blocks.length - 1] + const startDate = dayjs.utc(firstBlock.header.timestamp).endOf('day') + const endDate = dayjs.utc(lastBlock.header.timestamp).endOf('day') + + let dates: Date[] = [] + for (let date = startDate; !date.isAfter(endDate); date = date.add(1, 'day').endOf('day')) { + dates.push(date.toDate()) + } + + const ognDailyStats = [] as OGNDailyStat[] + + for (const date of dates) { + const dailyOGNStatInserts = await updateOGNDailyStats(ctx, date) + if (dailyOGNStatInserts) { + ognDailyStats.push(dailyOGNStatInserts.dailyStat) + } + } + + if (ctx.isHead) { + const updatedOGNStats = (await applyCoingeckoData(ctx, { + Entity: OGNDailyStat, + coinId: 'origin-protocol', + // startTimestamp: Date.UTC(2023, 4, 17), + })) as OGNDailyStat[] + + const existingOGNIds = ognDailyStats.map((stat) => stat.id) + ognDailyStats.push(...updatedOGNStats.filter((stat) => existingOGNIds.indexOf(stat.id) < 0)) + } + + await ctx.store.upsert(ognDailyStats) +} + +async function updateOGNDailyStats(ctx: Context, date: Date) { + const queryParams = { + where: { timestamp: LessThanOrEqual(date) }, + order: { timestamp: 'desc' as FindOptionsOrderValue }, + } + + const [stakedBalance, holdersOverThreshold, totalSupply] = await Promise.all([ + ctx.store.findOne(ERC20Balance, { + where: { + timestamp: LessThanOrEqual(date), + address: '0x8207c1ffc5b6804f6024322ccf34f29c3541ae26', + account: '0x63898b3b6ef3d39332082178656e9862bee45c57', + }, + order: { timestamp: 'desc' as FindOptionsOrderValue }, + }), + ctx.store.countBy(ERC20Holder, { + balance: MoreThanOrEqual(10n ** 20n), + address: '0x8207c1ffc5b6804f6024322ccf34f29c3541ae26', + }), // 100 OGN + ctx.store.findOne(ERC20State, { + where: { + timestamp: LessThanOrEqual(date), + address: '0x8207c1ffc5b6804f6024322ccf34f29c3541ae26', + }, + order: { timestamp: 'desc' as FindOptionsOrderValue }, + }), + ]) + + const allEntities = [totalSupply].filter(Boolean) + if (!totalSupply) { + return null + } + + const mostRecentEntity = allEntities.reduce((highest, current) => { + if (!highest || !current) return current + return current.blockNumber > highest.blockNumber ? current : highest + }) + + const id = date.toISOString().substring(0, 10) + + const dailyStat = new OGNDailyStat({ + id, + blockNumber: mostRecentEntity?.blockNumber, + timestamp: mostRecentEntity?.timestamp, + + totalSupply: totalSupply?.totalSupply || 0n, + totalStaked: stakedBalance?.balance || 0n, + totalSupplyUSD: 0, + tradingVolumeUSD: 0, + marketCapUSD: 0, + priceUSD: 0, + holdersOverThreshold, + }) + + return { dailyStat } +} diff --git a/src/model/generated/index.ts b/src/model/generated/index.ts index 1a746a59..1b352022 100644 --- a/src/model/generated/index.ts +++ b/src/model/generated/index.ts @@ -60,6 +60,7 @@ export * from "./oethStrategyHoldingDailyStat.model" export * from "./oethCollateralDailyStat.model" export * from "./oethRewardTokenCollected.model" export * from "./oethWithdrawalRequest.model" +export * from "./ognDailyStat.model" export * from "./ogv.model" export * from "./ogvAddress.model" export * from "./ogvLockupTxLog.model" diff --git a/src/model/generated/ognDailyStat.model.ts b/src/model/generated/ognDailyStat.model.ts new file mode 100644 index 00000000..aa3618d5 --- /dev/null +++ b/src/model/generated/ognDailyStat.model.ts @@ -0,0 +1,40 @@ +import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, IntColumn as IntColumn_, Index as Index_, DateTimeColumn as DateTimeColumn_, BigIntColumn as BigIntColumn_, FloatColumn as FloatColumn_} from "@subsquid/typeorm-store" + +@Entity_() +export class OGNDailyStat { + constructor(props?: Partial) { + Object.assign(this, props) + } + + @PrimaryColumn_() + id!: string + + @Index_() + @IntColumn_({nullable: false}) + blockNumber!: number + + @Index_() + @DateTimeColumn_({nullable: false}) + timestamp!: Date + + @BigIntColumn_({nullable: false}) + totalSupply!: bigint + + @FloatColumn_({nullable: false}) + totalSupplyUSD!: number + + @BigIntColumn_({nullable: false}) + totalStaked!: bigint + + @FloatColumn_({nullable: false}) + tradingVolumeUSD!: number + + @FloatColumn_({nullable: false}) + marketCapUSD!: number + + @FloatColumn_({nullable: false}) + priceUSD!: number + + @IntColumn_({nullable: false}) + holdersOverThreshold!: number +} diff --git a/src/utils/coingecko.ts b/src/utils/coingecko.ts index 44e960f8..74c0cb56 100644 --- a/src/utils/coingecko.ts +++ b/src/utils/coingecko.ts @@ -1,12 +1,16 @@ import { Between, LessThanOrEqual } from 'typeorm' import { parseEther } from 'viem' -import { OETHDailyStat, OGVDailyStat, OUSDDailyStat } from '@model' +import { OETHDailyStat, OGNDailyStat, OGVDailyStat, OUSDDailyStat } from '@model' import { Context } from '@processor' import { queryClient } from '@utils/queryClient' import { EntityClassT } from '@utils/type' -type DailyStat = EntityClassT | EntityClassT | EntityClassT +type DailyStat = + | EntityClassT + | EntityClassT + | EntityClassT + | EntityClassT export interface CoingeckoDataInput { prices: [number, number][] @@ -81,7 +85,7 @@ export async function applyCoingeckoData( let whereClause = { timestamp: LessThanOrEqual(getStartOfDayTimestamp()), } as any - if (Entity === OGVDailyStat) { + if (Entity === OGVDailyStat || Entity === OGNDailyStat) { whereClause.priceUSD = 0 } else { whereClause.pegPrice = 0n @@ -104,7 +108,9 @@ export async function applyCoingeckoData( if (response.status === 429) { throw new Error('Coingecko rate limited') } - return await response.json() + const result = await response.json() + console.log(`Found ${result.prices.length} prices`) + return result }, staleTime: 600_000, // 10 minutes @@ -115,13 +121,17 @@ export async function applyCoingeckoData( } else { const coingeckData = processCoingeckoData(coingeckoJson) for (const dayId in coingeckData) { - const stat = statsWithNoPrice.find((s) => s.id === dayId) as OETHDailyStat | OUSDDailyStat | OGVDailyStat + const stat = statsWithNoPrice.find((s) => s.id === dayId) as + | OETHDailyStat + | OUSDDailyStat + | OGVDailyStat + | OGNDailyStat const day = coingeckData[dayId] if (stat && day.prices) { stat.tradingVolumeUSD = day.total_volumes || 0 stat.marketCapUSD = day.market_caps || 0 - if (stat instanceof OGVDailyStat) { + if (stat instanceof OGVDailyStat || stat instanceof OGNDailyStat) { stat.priceUSD = day.prices } else { stat.pegPrice = parseEther(String(day.prices))