diff --git a/.github/workflows/on-main.yaml b/.github/workflows/on-main.yaml index d4673fd91..7dde3d9b4 100644 --- a/.github/workflows/on-main.yaml +++ b/.github/workflows/on-main.yaml @@ -4,6 +4,7 @@ on: push: branches: - main + - graph-watcher jobs: lint: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index e7dbf15eb..2b6ec2725 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -15,9 +15,8 @@ jobs: - name: Docker Login to Registry run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u vulcanize --password-stdin - name: Docker Pull - run: docker pull ghcr.io/vulcanize/watcher-ts:${{steps.vars.outputs.sha}} + run: docker pull ghcr.io/vulcanize/graph-watcher-ts:${{steps.vars.outputs.sha}} - name: Tag docker image - run: docker tag ghcr.io/vulcanize/watcher-ts:${{steps.vars.outputs.sha}} ghcr.io/vulcanize/watcher-ts:${{steps.vars.outputs.tag}} + run: docker tag ghcr.io/vulcanize/graph-watcher-ts:${{steps.vars.outputs.sha}} ghcr.io/vulcanize/graph-watcher-ts:${{steps.vars.outputs.tag}} - name: Docker Push to Github Hub - run: docker push ghcr.io/vulcanize/watcher-ts:${{steps.vars.outputs.tag}} - + run: docker push ghcr.io/vulcanize/graph-watcher-ts:${{steps.vars.outputs.tag}} diff --git a/packages/erc721-watcher/README.md b/packages/erc721-watcher/README.md index dfda3232e..497fea14e 100644 --- a/packages/erc721-watcher/README.md +++ b/packages/erc721-watcher/README.md @@ -8,6 +8,12 @@ yarn ``` +* Run the IPFS (go-ipfs version 0.12.2) daemon: + + ```bash + ipfs daemon + ``` + * Create a postgres12 database for the watcher: ```bash @@ -37,9 +43,9 @@ * The following core services should be setup and running on localhost: - * `vulcanize/go-ethereum` [v1.10.18-statediff-3.2.2](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-3.2.2) on port 8545 + * `vulcanize/go-ethereum` [v1.10.18-statediff-4.0.2-alpha](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-4.0.2-alpha) on port 8545 - * `vulcanize/ipld-eth-server` [v3.2.2](https://github.com/vulcanize/ipld-eth-server/releases/tag/v3.2.2) with native GQL API enabled, on port 8082 + * `vulcanize/ipld-eth-server` [v4.0.3-alpha](https://github.com/vulcanize/ipld-eth-server/releases/tag/v4.0.3-alpha) with native GQL API enabled, on port 8082 * In the [config file](./environments/local.toml): @@ -47,157 +53,7 @@ * Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint. -## Demo - -* Deploy an ERC721 token: - - ```bash - yarn nft:deploy - # NFT deployed to: NFT_ADDRESS - ``` - - Export the address of the deployed token to a shell variable for later use: - - ```bash - export NFT_ADDRESS="" - ``` - -* Open `http://localhost:3006/graphql` (GraphQL Playground) in a browser window - -* Connect MetaMask to `http://localhost:8545` (with chain ID `41337`) - -* Add a second account to Metamask and export the account address to a shell variable for later use: - - ```bash - export RECIPIENT_ADDRESS="" - ``` - -* To get the current block hash at any time, run: - - ```bash - yarn block:latest - ``` - -* Run the following GQL query (`eth_call`) in generated watcher graphql endpoint http://127.0.0.1:3006/graphql - - ```graphql - query { - name( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - ) { - value - proof { - data - } - } - symbol( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - ) { - value - proof { - data - } - } - balanceOf( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - owner: "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc" - ) { - value - proof { - data - } - } - } - ``` - -* Run the following GQL query (`storage`) in generated watcher graphql endpoint http://127.0.0.1:3006/graphql - - ```graphql - query { - _name( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - ) { - value - proof { - data - } - } - _symbol( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - ) { - value - proof { - data - } - } - _balances( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - key0: "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc" - ) { - value - proof { - data - } - } - } - ``` - -* Mint token - - ```bash - yarn nft:mint --nft $NFT_ADDRESS --to 0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc --token-id 1 - ``` - -* Get the latest blockHash and run the following query for `balanceOf` and `ownerOf` (`eth_call`): - - ```graphql - query { - fromBalanceOf: balanceOf( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - owner: "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc" - ) { - value - proof { - data - } - } - toBalanceOf: balanceOf( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - owner: "RECIPIENT_ADDRESS" - ) { - value - proof { - data - } - } - ownerOf( - blockHash: "LATEST_BLOCK_HASH" - contractAddress: "NFT_ADDRESS" - tokenId: 1 - ) { - value - proof { - data - } - } - } - ``` - - * Transfer token - - ```bash - yarn nft:transfer --nft $NFT_ADDRESS --from 0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc --to $RECIPIENT_ADDRESS --token-id 1 - ``` - - * Get the latest blockHash and replace the blockHash in the above query. The result should be different and the token should be transferred to the recipient. + * Update the `server` config with state checkpoint settings and provide the IPFS API address. ## Customize @@ -217,6 +73,8 @@ ## Run +Follow the steps below or follow the [Demo](./demo.md) + * Run the watcher: ```bash diff --git a/packages/erc721-watcher/demo.md b/packages/erc721-watcher/demo.md new file mode 100644 index 000000000..335b08807 --- /dev/null +++ b/packages/erc721-watcher/demo.md @@ -0,0 +1,264 @@ +# Demo + +* For setup follow the [steps in Readme](./README.md#setup). + +* Run the watcher: + + ```bash + yarn server + ``` + +* Run the job-runner: + + ```bash + yarn job-runner + ``` + +* Deploy an ERC721 token: + + ```bash + yarn nft:deploy + # NFT deployed to: NFT_ADDRESS + ``` + + Export the address of the deployed token to a shell variable for later use: + + ```bash + export NFT_ADDRESS="" + ``` + +* Run the following GQL mutation in generated watcher GraphQL endpoint http://127.0.0.1:3006/graphql + + ```graphql + mutation { + watchContract( + address: "NFT_ADDRESS" + kind: "ERC721" + checkpoint: true + ) + } + ``` + +* Connect MetaMask to `http://localhost:8545` (with chain ID `41337`) + +* Add a second account to Metamask and export the account address to a shell variable for later use: + + ```bash + export RECIPIENT_ADDRESS="" + ``` + +* To get the current block hash at any time, run: + + ```bash + yarn block:latest + ``` + +* Run the following GQL query (`eth_call`) in generated watcher GraphQL endpoint http://127.0.0.1:3006/graphql + + ```graphql + query { + name( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + symbol( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + balanceOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + owner: "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc" + ) { + value + proof { + data + } + } + } + ``` + +* Run the following GQL query (`storage`) in generated watcher GraphQL endpoint http://127.0.0.1:3006/graphql + + ```graphql + query { + _name( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + _symbol( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + ) { + value + proof { + data + } + } + _balances( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + key0: "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc" + ) { + value + proof { + data + } + } + } + ``` + +* Run the following GQL subscription in generated watcher GraphQL endpoint: + + ```graphql + subscription { + onEvent { + event { + __typename + ... on TransferEvent { + from + to + tokenId + }, + ... on ApprovalEvent { + owner + approved + tokenId + } + }, + block { + number + hash + } + } + } + ``` + +* Mint token + + ```bash + yarn nft:mint --nft $NFT_ADDRESS --to 0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc --token-id 1 + ``` + + * A Transfer event to 0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc shall be visible in the subscription at endpoint. + + * An auto-generated `diff_staged` IPLDBlock should be added with parent CID pointing to the initial checkpoint IPLDBlock. + + * Custom property `transferCount` should be 1 initially. + +* Run the getState query at the endpoint to get the latest IPLDBlock for NFT_ADDRESS: + + ```graphql + query { + getState ( + blockHash: "EVENT_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + # kind: "checkpoint" + # kind: "diff" + kind: "diff_staged" + ) { + cid + block { + cid + hash + number + timestamp + parentHash + } + contractAddress + data + } + } + ``` + + * `diff` IPLDBlocks get created corresponding to the `diff_staged` blocks when their respective eth_blocks reach the pruned region. + + * `data` contains the default state and also the custom state property `transferCount` that is indexed in [hooks.ts](./src/hooks.ts) file. + +* Get the latest blockHash and run the following query for `balanceOf` and `ownerOf` (`eth_call`): + + ```graphql + query { + fromBalanceOf: balanceOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + owner: "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc" + ) { + value + proof { + data + } + } + toBalanceOf: balanceOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + owner: "RECIPIENT_ADDRESS" + ) { + value + proof { + data + } + } + ownerOf( + blockHash: "LATEST_BLOCK_HASH" + contractAddress: "NFT_ADDRESS" + tokenId: 1 + ) { + value + proof { + data + } + } + } + ``` + +* Transfer token + + ```bash + yarn nft:transfer --nft $NFT_ADDRESS --from 0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc --to $RECIPIENT_ADDRESS --token-id 1 + ``` + + * An Approval event for ZERO_ADDRESS (0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc) shall be visible in the subscription at endpoint. + + * A Transfer event to $RECIPIENT_ADDRESS shall be visible in the subscription at endpoint. + + * An auto-generated `diff_staged` IPLDBlock should be added with parent CID pointing to the previous IPLDBlock. + + * Custom property `transferCount` should be incremented after transfer. This can be checked in the getState query and in IPFS webUI mentioned in the later steps. + +* Get the latest blockHash and replace the blockHash in the above query. The result should be different and the token should be transferred to the recipient. + +* Run the getState query again at the endpoint with the event blockHash. + +* After the `diff` block has been created (can check if event block number pruned in yarn server log), create a checkpoint using CLI in `packages/erc721-watcher`: + + ```bash + yarn checkpoint --address $NFT_ADDRESS + ``` + + * Run the getState query again with the output blockHash and kind checkpoint at the endpoint. + + * The latest checkpoint should have the aggregate of state diffs since the last checkpoint. + + * The IPLDBlock entries can be seen in pg-admin in table ipld_block. + +* All the diff and checkpoint IPLDBlocks should pushed to IPFS. + +* Open IPFS WebUI http://127.0.0.1:5001/webui and search for IPLDBlocks using their CIDs. + +* The state should have auto indexed data and also custom property `transferCount` according to code in [hooks](./src/hooks.ts) file `handleEvent` method. diff --git a/packages/erc721-watcher/environments/local.toml b/packages/erc721-watcher/environments/local.toml index 9b6e1e7f0..1e7fe5ddf 100644 --- a/packages/erc721-watcher/environments/local.toml +++ b/packages/erc721-watcher/environments/local.toml @@ -10,7 +10,7 @@ checkpointInterval = 2000 # IPFS API address (can be taken from the output on running the IPFS daemon). - # ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" + ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" [database] diff --git a/packages/erc721-watcher/src/database.ts b/packages/erc721-watcher/src/database.ts index 877e021a4..b63511271 100644 --- a/packages/erc721-watcher/src/database.ts +++ b/packages/erc721-watcher/src/database.ts @@ -3,7 +3,7 @@ // import assert from 'assert'; -import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, FindManyOptions } from 'typeorm'; +import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, FindManyOptions, FindOneOptions } from 'typeorm'; import path from 'path'; import { IPLDDatabase as BaseDatabase, IPLDDatabaseInterface, QueryOptions, StateKind, Where } from '@vulcanize/util'; @@ -28,6 +28,7 @@ import { _Owners } from './entity/_Owners'; import { _Balances } from './entity/_Balances'; import { _TokenApprovals } from './entity/_TokenApprovals'; import { _OperatorApprovals } from './entity/_OperatorApprovals'; +import { TransferCount } from './entity/TransferCount'; export class Database implements IPLDDatabaseInterface { _config: ConnectionOptions; @@ -128,6 +129,35 @@ export class Database implements IPLDDatabaseInterface { }); } + async getTransferCount (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + const repo = queryRunner.manager.getRepository(TransferCount); + const whereOptions: FindConditions = { id }; + + if (blockHash) { + whereOptions.blockHash = blockHash; + } + + const findOptions = { + where: whereOptions, + order: { + blockNumber: 'DESC' + } + }; + + let entity = await repo.findOne(findOptions as FindOneOptions); + + if (!entity && findOptions.where.blockHash) { + entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); + } + + return entity; + } + + async saveTransferCount (queryRunner: QueryRunner, transferCount: TransferCount): Promise { + const repo = queryRunner.manager.getRepository(TransferCount); + return repo.save(transferCount); + } + async _getName ({ blockHash, contractAddress }: { blockHash: string, contractAddress: string }): Promise<_Name | undefined> { return this._conn.getRepository(_Name) .findOne({ diff --git a/packages/erc721-watcher/src/entity/TransferCount.ts b/packages/erc721-watcher/src/entity/TransferCount.ts new file mode 100644 index 000000000..681ee91af --- /dev/null +++ b/packages/erc721-watcher/src/entity/TransferCount.ts @@ -0,0 +1,20 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; + +@Entity() +export class TransferCount { + @PrimaryColumn('varchar') + id!: string; + + @PrimaryColumn('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('integer') + count!: number; +} diff --git a/packages/erc721-watcher/src/hooks.ts b/packages/erc721-watcher/src/hooks.ts index 09298b7c8..22e6382e2 100644 --- a/packages/erc721-watcher/src/hooks.ts +++ b/packages/erc721-watcher/src/hooks.ts @@ -4,9 +4,10 @@ import assert from 'assert'; -// import { updateStateForMappingType, updateStateForElementaryType } from '@vulcanize/util'; +import { updateStateForMappingType, updateStateForElementaryType } from '@vulcanize/util'; import { Indexer, ResultEvent } from './indexer'; +import { TransferCount } from './entity/TransferCount'; /** * Hook function to store an initial state. @@ -75,6 +76,69 @@ export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Pr assert(indexer); assert(eventData); - // Use indexer methods to index data. - // Pass `diff` parameter to indexer methods as true to save an auto-generated state from the indexed data. + // Perform indexing based on the type of event. + switch (eventData.event.__typename) { + case 'TransferEvent': { + // Get event fields from eventData. + const { from, to, tokenId } = eventData.event; + + // Update balance entry for the sender in database. + if (from !== '0x0000000000000000000000000000000000000000') { + await indexer._balances(eventData.block.hash, eventData.contract, from, true); + } + + // Update balance entry for the receiver in database. + if (to !== '0x0000000000000000000000000000000000000000') { + await indexer._balances(eventData.block.hash, eventData.contract, to, true); + } + + // Update owner for the tokenId in database. + await indexer._owners(eventData.block.hash, eventData.contract, tokenId, true); + + // Code to update a custom state property transferCount. + // { + // "transferCount": "1" + // } + // Fetch transferCount entity from database. + let transferCount = await indexer.transferCount(eventData.block.hash, eventData.contract); + + if (!transferCount) { + transferCount = new TransferCount(); + transferCount.blockHash = eventData.block.hash; + transferCount.blockNumber = eventData.block.number; + transferCount.id = eventData.contract; + transferCount.count = 0; + } + + // Increment count on transfer event. + transferCount.count++; + + // Update state for custom property transferCount. + const stateUpdate = updateStateForElementaryType({}, 'transferCount', transferCount.count); + await indexer.createDiffStaged(eventData.contract, eventData.block.hash, stateUpdate); + + // Save transferCount to database. + await indexer.saveOrUpdateTransferCount(transferCount); + + break; + } + case 'ApprovalEvent': { + // Get event fields from eventData. + const { tokenId } = eventData.event; + + // Update tokenApprovals for the tokenId in database. + await indexer._tokenApprovals(eventData.block.hash, eventData.contract, tokenId, true); + + break; + } + case 'ApprovalForAllEvent': { + // Get event fields from eventData. + const { owner, operator } = eventData.event; + + // Update operatorApprovals for the tokenId in database. + await indexer._operatorApprovals(eventData.block.hash, eventData.contract, owner, operator, true); + + break; + } + } } diff --git a/packages/erc721-watcher/src/indexer.ts b/packages/erc721-watcher/src/indexer.ts index 38f671ea4..14b07499a 100644 --- a/packages/erc721-watcher/src/indexer.ts +++ b/packages/erc721-watcher/src/indexer.ts @@ -40,6 +40,7 @@ import { SyncStatus } from './entity/SyncStatus'; import { IpldStatus } from './entity/IpldStatus'; import { BlockProgress } from './entity/BlockProgress'; import { IPLDBlock } from './entity/IPLDBlock'; +import { TransferCount } from './entity/TransferCount'; const log = debug('vulcanize:indexer'); @@ -430,6 +431,39 @@ export class Indexer implements IPLDIndexerInterface { return result; } + async transferCount (blockHash: string, contractAddress: string): Promise { + const dbTx = await this._db.createTransactionRunner(); + let res; + + try { + res = await this._db.getTransferCount(dbTx, { id: contractAddress, blockHash }); + await dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + + return res; + } + + async saveOrUpdateTransferCount (transferCount: TransferCount) { + const dbTx = await this._db.createTransactionRunner(); + let res; + + try { + await this._db.saveTransferCount(dbTx, transferCount); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + + return res; + } + async _name (blockHash: string, contractAddress: string, diff = false): Promise { const entity = await this._db._getName({ blockHash, contractAddress }); if (entity) { diff --git a/packages/util/src/ipld-helper.ts b/packages/util/src/ipld-helper.ts index b21706a62..4e46e8817 100644 --- a/packages/util/src/ipld-helper.ts +++ b/packages/util/src/ipld-helper.ts @@ -1,13 +1,13 @@ import _ from 'lodash'; -export const updateStateForElementaryType = (initialObject: any, stateVariable: string, value: string): any => { +export const updateStateForElementaryType = (initialObject: any, stateVariable: string, value: any): any => { const object = _.cloneDeep(initialObject); const path = ['state', stateVariable]; return _.set(object, path, value); }; -export const updateStateForMappingType = (initialObject: any, stateVariable: string, keys: string[], value: string): any => { +export const updateStateForMappingType = (initialObject: any, stateVariable: string, keys: string[], value: any): any => { const object = _.cloneDeep(initialObject); keys.unshift('state', stateVariable);