From fa12a66291c4c4ae98c9f8da1a37eccd77c39617 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 23 Apr 2024 13:39:00 +0800 Subject: [PATCH 1/7] Create troubleshooting.md --- docs/troubleshooting.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/troubleshooting.md diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1 @@ + From 82b32becffeb987fe313d77e8840d886bbf8b64d Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 23 Apr 2024 13:49:27 +0800 Subject: [PATCH 2/7] devops --- docs/devops.md | 57 +++++++++++++++++++++++++++++++++++++++++ docs/troubleshooting.md | 1 - 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 docs/devops.md delete mode 100644 docs/troubleshooting.md diff --git a/docs/devops.md b/docs/devops.md new file mode 100644 index 00000000..f729d5b5 --- /dev/null +++ b/docs/devops.md @@ -0,0 +1,57 @@ +## Application Overview + +The application is deployed as **indexer-v2** on Fly under the [Gitcoin organization](https://fly.io/dashboard/gtc). If you do not have access to this dashboard, please request permission. + +## Fly CLI Installation + +Before deploying or managing the application, ensure you have the Fly CLI installed. Follow these steps to install the Fly CLI: + +1. Download the Fly CLI from [Fly's official documentation](https://fly.io/docs/getting-started/installing-flyctl/). +2. Install the CLI following the instructions for your operating system. +3. Once installed, run `flyctl auth login` to authenticate with your Fly account. + +## Fly CLI Usage + +:warn: Always run Fly commands with an explicit config option, for example: + +``` +fly -c fly.production.toml status +``` + +## Components and Architecture + +The [application](../fly.production.toml) contains two main components organized into separate process groups: + +- **web**: Handles client requests and must scale horizontally to manage varying loads effectively. +- **indexer**: Responsible for updating and maintaining the database. Only one active instance runs due to its role as the sole writer to prevent data conflicts. + +### Why Separate Process Groups? + +- **Decoupling**: Separating the indexer and web processes allows each to be scaled and managed according to its specific needs. +- **Scalability**: The web service can scale horizontally across multiple instances to accommodate increased traffic without affecting the indexer. +- **Consistency**: The indexer, as a single writer, ensures data consistency and integrity, critical for the database's health and performance. + +## Deployment Process + +Check the [Github Workflow](../.github/workflows/deploy-branch.yml) to understand how the deployment works. + +### General Recommendation + +- Always deploy through the GitHub workflow to ensure all changes are tracked and reversible. +- Avoid deploying directly from the CLI as it will deploy your working directory and changes might be deployed accidentally. + +### Deployment Checks + +- Use `fly status` to check the status of the instances. +- Although two Indexer instances are running, only one actively indexes due to a locking mechanism. + +## Lock Mechanism and High Availability + +- A write lock ensures that while multiple indexer instances may be present for failover readiness, only one actively indexes at any time. +- This is especially crucial during updates: + - If updating from v50 to v51, two v50 instances may run; one indexes while the other remains idle. + - Deploying v51 will stop an instance of v50 and start v51, maintaining continuous indexing and high availability. + +### Fly Auto-Scaling + +There are a couple of web instances provisioned, the stopped ones are on stand-by. Fly automatically starts and stops machines based on load. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md deleted file mode 100644 index 8b137891..00000000 --- a/docs/troubleshooting.md +++ /dev/null @@ -1 +0,0 @@ - From 1c8ac848862a22aa3bb31c1bae327e16e1fffffb Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 23 Apr 2024 13:50:32 +0800 Subject: [PATCH 3/7] Update devops.md --- docs/devops.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/devops.md b/docs/devops.md index f729d5b5..0804d97e 100644 --- a/docs/devops.md +++ b/docs/devops.md @@ -12,7 +12,7 @@ Before deploying or managing the application, ensure you have the Fly CLI instal ## Fly CLI Usage -:warn: Always run Fly commands with an explicit config option, for example: +⚠️ Always run Fly commands with an explicit config option, for example: ``` fly -c fly.production.toml status From ce2abf5c789ea29812aae6ed2100311d09e27414 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 23 Apr 2024 13:58:55 +0800 Subject: [PATCH 4/7] docs --- docs/devops.md | 60 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/docs/devops.md b/docs/devops.md index 0804d97e..56bddfab 100644 --- a/docs/devops.md +++ b/docs/devops.md @@ -1,8 +1,8 @@ -## Application Overview +# Application Overview The application is deployed as **indexer-v2** on Fly under the [Gitcoin organization](https://fly.io/dashboard/gtc). If you do not have access to this dashboard, please request permission. -## Fly CLI Installation +# Fly CLI Installation Before deploying or managing the application, ensure you have the Fly CLI installed. Follow these steps to install the Fly CLI: @@ -10,7 +10,7 @@ Before deploying or managing the application, ensure you have the Fly CLI instal 2. Install the CLI following the instructions for your operating system. 3. Once installed, run `flyctl auth login` to authenticate with your Fly account. -## Fly CLI Usage +# Fly CLI Usage ⚠️ Always run Fly commands with an explicit config option, for example: @@ -18,7 +18,7 @@ Before deploying or managing the application, ensure you have the Fly CLI instal fly -c fly.production.toml status ``` -## Components and Architecture +# Components and Architecture The [application](../fly.production.toml) contains two main components organized into separate process groups: @@ -31,7 +31,7 @@ The [application](../fly.production.toml) contains two main components organized - **Scalability**: The web service can scale horizontally across multiple instances to accommodate increased traffic without affecting the indexer. - **Consistency**: The indexer, as a single writer, ensures data consistency and integrity, critical for the database's health and performance. -## Deployment Process +# Deployment Process Check the [Github Workflow](../.github/workflows/deploy-branch.yml) to understand how the deployment works. @@ -52,6 +52,54 @@ Check the [Github Workflow](../.github/workflows/deploy-branch.yml) to understan - If updating from v50 to v51, two v50 instances may run; one indexes while the other remains idle. - Deploying v51 will stop an instance of v50 and start v51, maintaining continuous indexing and high availability. -### Fly Auto-Scaling +# Operational Tasks + +## Logs + +Logs are shipped to [Datadog](https://app.datadoghq.eu/logs). + +You can also check the latest logs with the [Fly logs](https://fly.io/docs/flyctl/logs/) command. + +## Scaling + +### Fly Web Auto-Scaling There are a couple of web instances provisioned, the stopped ones are on stand-by. Fly automatically starts and stops machines based on load. + +### Horizontal Scaling + +Use the `scale` command to scale up and down the number of machines: + + +``` +flyctl -c fly.production.toml scale web=5 +``` + +### Vertical Scaling + +Show the current CPU and RAM configuration of the machines: + +``` +flyctl -c fly.production.toml scale show +``` + +Check the available VM size presets: + +``` +flyctl platform vm-sizes +``` + +Change the VM size: + + +``` +flyctl -c fly.production.toml scale vm performance-1x +``` + +Change memory: + +``` +flyctl -c fly.production.toml scale memory 2048 +``` + + From 86449b17103fbac60051d1230305de968e126d26 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 23 Apr 2024 14:00:46 +0800 Subject: [PATCH 5/7] docs --- docs/devops.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/devops.md b/docs/devops.md index 56bddfab..ceeba155 100644 --- a/docs/devops.md +++ b/docs/devops.md @@ -1,5 +1,3 @@ -# Application Overview - The application is deployed as **indexer-v2** on Fly under the [Gitcoin organization](https://fly.io/dashboard/gtc). If you do not have access to this dashboard, please request permission. # Fly CLI Installation From ceed89ed50ca1af40ae2ec368ecda82e920aed7e Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 23 Apr 2024 14:01:57 +0800 Subject: [PATCH 6/7] fix checkpr --- .github/workflows/check-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml index 3351057f..6481098d 100644 --- a/.github/workflows/check-pr.yml +++ b/.github/workflows/check-pr.yml @@ -17,7 +17,7 @@ jobs: # (ab)use the development environment until we build on github - name: Build and test run: | - flyctl -c fly.development.toml deploy --remote-only --build-only + flyctl -c fly.staging.toml deploy --remote-only --build-only env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} From d05fbade250407c6bec3dd33668ab80fa2961ae1 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 24 Apr 2024 12:52:22 +0800 Subject: [PATCH 7/7] split event handlers --- src/index.ts | 16 ++++-- .../ProjectRegistry/V2/ProjectCreated.ts | 57 +++++++++++++++++++ src/indexer/allo/v1/handleEvent.ts | 44 -------------- src/indexer/allo/v1/project.ts | 12 ++++ tsconfig.json | 5 +- 5 files changed, 85 insertions(+), 49 deletions(-) create mode 100644 src/indexer/allo/handlers/AlloV1/ProjectRegistry/V2/ProjectCreated.ts create mode 100644 src/indexer/allo/v1/project.ts diff --git a/src/index.ts b/src/index.ts index 6c6f19e1..c45864f0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -506,11 +506,19 @@ async function catchupAndWatchChain( }); }); } else { - const handler = args.event.contractName.startsWith("AlloV1") - ? handleAlloV1Event - : handleAlloV2Event; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const module = await import( + `./indexe/handlers/${args.event.contractName}/${args.event.name}` + ); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const changesets = module.default(args); + + // const handler = args.event.contractName.startsWith("AlloV1") + // ? handleAlloV1Event + // : handleAlloV2Event; - const changesets = await handler(args); + // const changesets = await handler(args); for (const changeset of changesets) { await db.applyChange(changeset); diff --git a/src/indexer/allo/handlers/AlloV1/ProjectRegistry/V2/ProjectCreated.ts b/src/indexer/allo/handlers/AlloV1/ProjectRegistry/V2/ProjectCreated.ts new file mode 100644 index 00000000..65bfb539 --- /dev/null +++ b/src/indexer/allo/handlers/AlloV1/ProjectRegistry/V2/ProjectCreated.ts @@ -0,0 +1,57 @@ +import { EventHandlerArgs } from "chainsauce"; +import { Indexer } from "@/indexer/indexer.js"; +import { Changeset } from "@/database/index.js"; +import { fullProjectId } from "@/indexer/allo/v1/project.js"; +import { parseAddress } from "@/address.js"; + +export default async function ( + args: EventHandlerArgs +): Promise { + const { + chainId, + event, + context: { rpcClient }, + } = args; + + const projectId = fullProjectId( + chainId, + Number(event.params.projectID), + event.address + ); + + const tx = await rpcClient.getTransaction({ + hash: event.transactionHash, + }); + + const createdBy = tx.from; + + return [ + { + type: "InsertProject", + project: { + tags: ["allo-v1"], + chainId, + registryAddress: parseAddress(event.address), + id: projectId, + name: "", + projectNumber: Number(event.params.projectID), + metadataCid: null, + metadata: null, + createdByAddress: parseAddress(createdBy), + createdAtBlock: event.blockNumber, + updatedAtBlock: event.blockNumber, + projectType: "canonical", + }, + }, + { + type: "InsertProjectRole", + projectRole: { + chainId, + projectId, + address: parseAddress(event.params.owner), + role: "owner", + createdAtBlock: event.blockNumber, + }, + }, + ]; +} diff --git a/src/indexer/allo/v1/handleEvent.ts b/src/indexer/allo/v1/handleEvent.ts index bc30b9fe..2879fbfa 100644 --- a/src/indexer/allo/v1/handleEvent.ts +++ b/src/indexer/allo/v1/handleEvent.ts @@ -67,50 +67,6 @@ export async function handleEvent( switch (event.name) { // -- PROJECTS - case "ProjectCreated": { - const projectId = fullProjectId( - chainId, - Number(event.params.projectID), - event.address - ); - - const tx = await rpcClient.getTransaction({ - hash: event.transactionHash, - }); - - const createdBy = tx.from; - - return [ - { - type: "InsertProject", - project: { - tags: ["allo-v1"], - chainId, - registryAddress: parseAddress(event.address), - id: projectId, - name: "", - projectNumber: Number(event.params.projectID), - metadataCid: null, - metadata: null, - createdByAddress: parseAddress(createdBy), - createdAtBlock: event.blockNumber, - updatedAtBlock: event.blockNumber, - projectType: "canonical", - }, - }, - { - type: "InsertProjectRole", - projectRole: { - chainId, - projectId, - address: parseAddress(event.params.owner), - role: "owner", - createdAtBlock: event.blockNumber, - }, - }, - ]; - } - case "MetadataUpdated": { const projectId = fullProjectId( chainId, diff --git a/src/indexer/allo/v1/project.ts b/src/indexer/allo/v1/project.ts new file mode 100644 index 00000000..0e3f4a32 --- /dev/null +++ b/src/indexer/allo/v1/project.ts @@ -0,0 +1,12 @@ +import { ethers } from "ethers"; + +export function fullProjectId( + projectChainId: number, + projectId: number, + projectRegistryAddress: string +) { + return ethers.utils.solidityKeccak256( + ["uint256", "address", "uint256"], + [projectChainId, projectRegistryAddress, projectId] + ); +} diff --git a/tsconfig.json b/tsconfig.json index 05e208a3..362d8a18 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,7 +14,10 @@ "rootDir": "./", "resolveJsonModule": true, "declaration": false, - "sourceMap": true + "sourceMap": true, + "paths": { + "@/*": ["src/*"], + } }, "exclude": ["node_modules", "dist"] }