Add demo for graph-watcher IPLD statediff and checkpointing

README.md

@@ -45,6 +45,10 @@ yarn build
 
 * [graph-node](./packages/graph-node/README.md)
 
+## Demos
+
+* [IPLD statediff and checkpointing](./ipld-demo.md)
+
 ## Services
 
 The default config files used by the watchers assume the following services are setup and running on localhost:

ipld-demo.md

@@ -0,0 +1,209 @@
+# Demo for IPLD statediff and checkpointing
+
+* In the root of `graph-watcher-ts`, run:
+
+  ```bash
+  yarn && yarn build
+  ```
+
+* In console, run the IPFS daemon:
+
+  ```bash
+  # Verify ipfs version
+  ipfs version
+  # ipfs version 0.12.2
+
+  ipfs daemon
+  ```
+
+* The following services should be running to work with watcher:
+
+  * [vulcanize/go-ethereum](https://github.com/vulcanize/go-ethereum) ([v1.10.17-statediff-3.2.0](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.17-statediff-3.2.0)) on port 8545.
+  * [vulcanize/ipld-eth-server](https://github.com/vulcanize/ipld-eth-server) ([v3.0.0](https://github.com/vulcanize/ipld-eth-server/releases/tag/v3.0.0)) with native GQL API enabled on port 8082 and RPC API enabled on port 8081.
+  * [postgraphile](https://github.com/vulcanize/postgraphile) ([v1.1.1](https://github.com/vulcanize/postgraphile/releases/tag/v1.1.1)) on the `vulcanize/ipld-eth-server` database, on port 5000
+
+* Deploy `Example` contract:
+
+  ```bash
+  cd packages/graph-node
+
+  yarn example:deploy
+  ```
+
+* Set the returned address to the variable `$EXAMPLE_ADDRESS`:
+
+  ```bash
+  EXAMPLE_ADDRESS=
+  ```
+
+* In `packages/graph-node`, run:
+
+  ```bash
+  cp .env.example .env
+  ```
+
+  * In `.env` file, set `EXAMPLE_CONTRACT_ADDRESS` to the `EXAMPLE_ADDRESS`.
+
+  * In [packages/graph-node/test/subgraph/example1/subgraph.yaml](./packages/graph-node/test/subgraph/example1/subgraph.yaml), set the source address for `Example1` datasource to the `EXAMPLE_ADDRESS`.
+
+  ```bash
+  yarn build:example
+  ```
+
+* In `packages/codegen`, create a `config.yaml` file with the following contents:
+
+  ```yaml
+  contracts:
+    - name: Example
+      path: ../graph-node/test/contracts/Example.sol
+      kind: Example1
+
+  outputFolder: ../demo-example-watcher
+  mode: all
+  kind: active
+  port: 3008
+  flatten: true
+  subgraphPath: ../graph-node/test/subgraph/example1/build
+  ```
+
+  Reference: [packages/codegen/README.md](./packages/codegen/README.md#run)
+
+* Generate watcher:
+
+  ```bash
+  cd packages/codegen
+
+  yarn codegen --config-file ./config.yaml
+  ```
+
+* In `packages/demo-example-watcher`, run:
+
+  ```bash
+  yarn
+  ```
+
+* Create dbs:
+
+  ```bash
+  sudo su - postgres
+  # Delete databases if they already exist.
+  dropdb demo-example-watcher
+  dropdb demo-example-watcher-job-queue
+
+  # Create databases
+  createdb demo-example-watcher
+  createdb demo-example-watcher-job-queue
+  ```
+
+  Enable the `pgcrypto` extension.
+  ```
+  psql -U postgres -h localhost demo-example-watcher-job-queue
+
+  demo-example-watcher-job-queue=# CREATE EXTENSION pgcrypto;
+  demo-example-watcher-job-queue=# exit
+  ```
+
+* In a new terminal, in `packages/demo-example-watcher`, run:
+
+  ```bash
+  yarn server
+  ```
+
+  ```bash
+  yarn job-runner
+  ```
+
+* Run the following GQL subscription at the [graphql endpoint](http://127.0.0.1:3008/graphql):
+
+  ```graphql
+  subscription {
+    onEvent {
+      event {
+        __typename
+        ... on TestEvent {
+          param1
+          param2
+          param3
+        },
+      },
+      block {
+        number
+        hash
+      }
+    }
+  }
+  ```
+
+* Trigger the `Test` event by calling example contract method:
+
+  ```bash
+  cd packages/graph-node
+
+  yarn example:test --address $EXAMPLE_ADDRESS
+  ```
+
+  A `Test` event shall be visible in the subscription at endpoint.
+
+* Run the `getState` query at the endpoint to get the latest `IPLDBlock` for `EXAMPLE_ADDRESS`:
+
+  ```graphql
+  query {
+    getState (
+      blockHash: "EVENT_BLOCK_HASH"
+      contractAddress: "EXAMPLE_ADDRESS"
+      kind: "diff_staged"
+    ) {
+      cid
+      block {
+        cid
+        hash
+        number
+        timestamp
+        parentHash
+      }
+      contractAddress
+      data
+    }
+  }
+  ```
+
+* Run the query for entity at the endpoint:
+
+  ```graphql
+  query {
+    author (
+      block: {
+        hash: "EVENT_BLOCK_HASH"
+      }
+      id: "0xdc7d7a8920c8eecc098da5b7522a5f31509b5bfc"
+    ) {
+      __typename
+      name
+      paramInt
+      paramBigInt
+      paramBytes
+    }
+  }
+  ```
+
+* `diff` IPLDBlocks get created corresponding to the `diff_staged` blocks when their respective `eth_block`s reach the pruned region.
+
+* In `packages/demo-example-watcher`:
+
+  * After the `diff` block has been created, create a `checkpoint`:
+
+    ```bash
+    cd packages/demo-example-watcher
+
+    yarn checkpoint --address $EXAMPLE_ADDRESS
+    ```
+
+    * A `checkpoint` IPLDBlock should be created at the latest canonical block hash.
+
+    * Run the `getState` query again at the endpoint with the output `blockHash` and kind `checkpoint`.
+
+* All the `IPLDBlock` entries can be seen in `pg-admin` in table `ipld_block`.
+
+* All the `diff` and `checkpoint` IPLDBlocks should be pushed to `IPFS`.
+
+* Open IPFS WebUI http://127.0.0.1:5001/webui and search for `IPLDBlock`s using their `CID`s.

packages/graph-node/hardhat.config.ts

@@ -5,6 +5,7 @@
 import '@nomiclabs/hardhat-waffle';
 
 import './test/tasks/example-deploy';
+import './test/tasks/example-test';
 
 // You need to export an object to set up your config
 // Go to https://hardhat.org/config/ to learn more

packages/graph-node/package.json

@@ -42,7 +42,8 @@
     "build:example": "cd test/subgraph/example1 && yarn && yarn codegen && yarn build",
     "watch": "DEBUG=vulcanize:* nodemon --watch src src/watcher.ts",
     "compare-entity": "node bin/compare-entity",
-    "example:deploy": "hardhat --network localhost example-deploy"
+    "example:deploy": "hardhat --network localhost example-deploy",
+    "example:test": "hardhat --network localhost example-test"
   },
   "dependencies": {
     "@apollo/client": "^3.3.19",

packages/graph-node/test/subgraph/example1/subgraph.yaml

@@ -8,7 +8,7 @@ dataSources:
     source:
       address: "0xD5567AFC3C6c1325698F27d97b74D9ea9c44295e"
       abi: Example1
-      startBlock: 100
+      startBlock: 10
     mapping:
       kind: ethereum/events
       apiVersion: 0.0.5

packages/graph-node/test/tasks/example-test.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { task, types } from 'hardhat/config';
+import '@nomiclabs/hardhat-ethers';
+import { ContractTransaction } from 'ethers';
+
+task('example-test', 'Trigger Test event')
+  .addParam('address', 'Contract address', undefined, types.string)
+  .setAction(async (args, hre) => {
+    const { address } = args;
+    await hre.run('compile');
+    const Example = await hre.ethers.getContractFactory('Example');
+    const example = Example.attach(address);
+
+    const transaction: ContractTransaction = await example.emitEvent();
+
+    const receipt = await transaction.wait();
+
+    if (receipt.events) {
+      const TestEvent = receipt.events.find(el => el.event === 'Test');
+
+      if (TestEvent && TestEvent.args) {
+        console.log('Test Event');
+        console.log('param1:', TestEvent.args.param1.toString());
+        console.log('param2:', TestEvent.args.param2.toString());
+        console.log('param3:', TestEvent.args.param3.toString());
+      }
+    }
+  });

packages/graph-test-watcher/package.json

@@ -32,25 +32,24 @@
     "@ethersproject/providers": "5.3.0",
     "@ipld/dag-cbor": "^6.0.12",
     "@vulcanize/cache": "^0.1.0",
+    "@vulcanize/graph-node": "^0.1.0",
     "@vulcanize/ipld-eth-client": "^0.1.0",
     "@vulcanize/solidity-mapper": "^0.1.0",
     "@vulcanize/util": "^0.1.0",
-    "@vulcanize/graph-node": "^0.1.0",
     "apollo-server-express": "^2.25.0",
     "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.2.0",
     "express": "^4.17.1",
     "graphql": "^15.5.0",
     "graphql-import-node": "^0.0.4",
-    "ipfs-http-client": "^53.0.1",
     "json-bigint": "^1.0.0",
     "lodash": "^4.17.21",
     "multiformats": "^9.4.8",
     "reflect-metadata": "^0.1.13",
     "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",

packages/util/package.json

@@ -12,7 +12,8 @@
     "lodash": "^4.17.21",
     "multiformats": "^9.4.8",
     "pg-boss": "^6.1.0",
-    "toml": "^3.0.0"
+    "toml": "^3.0.0",
+    "ipfs-http-client": "^56.0.3"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",

packages/util/src/ipfs.ts

@@ -12,6 +12,6 @@ export class IPFSClient {
   }
 
   async push (data: any): Promise<void> {
-    await this._client.dag.put(data, { format: 'dag-cbor', hashAlg: 'sha2-256' });
+    await this._client.dag.put(data, { storeCodec: 'dag-cbor', hashAlg: 'sha2-256' });
   }
 }