Merge pull request #23 from hack-a-chain-software/feat/kadena-indexer

feat(indexer): new package

.editorconfig

@@ -11,7 +11,7 @@ trim_trailing_whitespace = true
 [*.md]
 trim_trailing_whitespace = false
 
-[**.{json,vue,css,html,yml,ts,xml,yaml,yml,toml}]
+[**.{json,vue,css,html,yml,ts,js,xml,yaml,yml,toml,tf}]
 indent_style = space
 indent_size = 2
 

.github/workflows/dispatcher.yml

@@ -0,0 +1,18 @@
+name: Dispatch Commands
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  slashCommandDispatch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Dispatch received command
+        uses: peter-evans/slash-command-dispatch@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          reaction-token: ${{ secrets.GITHUB_TOKEN }}
+          issue-type: pull-request
+          commands: |
+            terraform-sync

.github/workflows/terraform.yml

@@ -0,0 +1,65 @@
+name: Terraform Deployment
+
+on:
+  repository_dispatch:
+    types: [terraform-sync-command]
+
+concurrency:
+  group: deploy-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  terraform:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure SSH
+        run: |
+          mkdir -p ~/.ssh/
+          echo "${{ secrets.TERRAFORM_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          echo "${{ secrets.TERRAFORM_SSH_PUB_KEY }}" > ~/.ssh/id_rsa.pub
+          chmod 600 ~/.ssh/id_rsa.pub
+          ssh-keyscan github.com >> ~/.ssh/known_hosts
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v1
+
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8.1.0
+
+      - name: Configure DigitalOcean credentials
+        env:
+          DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }}
+          DO_SECRET_ACCESS_KEY: ${{ secrets.DO_SECRET_ACCESS_KEY }}
+        run: |
+          aws configure --profile digitalocean set aws_access_key_id $DO_ACCESS_KEY
+          aws configure --profile digitalocean set aws_secret_access_key $DO_SECRET_ACCESS_KEY
+
+      - name: Create .tfvars file
+        run: |
+          echo "do_token=\"${{ secrets.DO_TOKEN }}\"" >> ./terraform/.tfvars
+          echo "project_id=\"${{ secrets.DO_PROJECT_ID }}\"" >> ./terraform/.tfvars
+          echo "" >> ./terraform/.tfvars
+
+      - name: Terraform Init
+        run: pnpm terraform install
+        env:
+          AWS_PROFILE: "digitalocean"
+
+      - name: Terraform Plan
+        run: pnpm terraform plan
+        env:
+          AWS_PROFILE: "digitalocean"
+
+      - name: Terraform Apply
+        run: pnpm terraform apply
+        env:
+          AWS_PROFILE: "digitalocean"

.gitignore

@@ -18,3 +18,6 @@ public/dist
 test-ledger
 wip
 /tests/*.js
+*.env
+**/.env
+**/node_modules

front/src/address.json

@@ -1,3 +1,3 @@
 {
     "TurboVerifier": "0x4130467f60DF72d84F86e38f2B4A3c18d818521d"
-}
+}

indexer/README.md

@@ -0,0 +1,24 @@
+# Indexer
+The "indexer" package is a set of services designed to index and track events on the Kadena blockchain. It provides a comprehensive solution for efficiently collecting, storing, and processing events, enabling subsequent analysis and queries.
+
+## Components
+The "indexer" consists of the following components:
+
+*PostgreSQL Database*: A relational database is used to store indexed events and other relevant information. The database schema is updated to filter and save only the desired modules in the database.
+
+*Chainweb-data*: This service is responsible for collecting events from the Kadena blockchain and inserting them into the database. The latest available image from the Docker website is used to run this service.
+
+*Chainweb-node*: This service runs a Chainweb node to receive events from the blockchain. Specific configuration is required to establish the connection between the "chainweb-data" service and the "chainweb" node.
+
+*Serveless*: A serveless component is used to provide a query interface to the indexed events in the database. This component allows for efficient queries and retrieval of the indexed data.
+
+## Infrastructure Management with Terraform
+The entire infrastructure for the "indexer" package is created and managed using Terraform. Terraform provides a declarative approach to infrastructure provisioning, allowing you to define and automate the deployment of resources needed for the indexer.
+
+With Terraform, you can define the desired state of your infrastructure using code, and Terraform will handle the provisioning and configuration of the necessary resources, such as the PostgreSQL database, Docker images, and serveless components. This ensures that your infrastructure remains consistent and reproducible across deployments.
+
+To manage the indexer infrastructure with Terraform, you can use the provided Terraform configuration files, which specify the required resources, their dependencies, and any necessary configurations. By running Terraform commands, such as terraform apply, you can create, update, or destroy the indexer infrastructure as needed.
+
+By leveraging Terraform for infrastructure management, you can easily scale, modify, and version your indexer infrastructure, providing a reliable and efficient foundation for indexing and analyzing events on the Kadena blockchain.
+
+![Indexer Graph](./assets/indexer.jpeg)

indexer/chainweb-data/.env.example

@@ -0,0 +1,11 @@
+# postgresql database
+POSTGRES_PASSWORD=""
+ADMIN_PASSWORD=""
+
+# Chainweb-data vars
+CWD_NODE=""
+CWD_DB_PORT=""
+CWD_DB_USER=""
+CWD_DB_NAME=""
+CWD_DB_PASS=""
+CWD_DB_HOST=""

indexer/chainweb-data/.gitignore

@@ -0,0 +1,23 @@
+!build/patches/**/node_modules
+!public/dist/_redirects
+*.cache
+*.log
+.DS_Store
+.eslintrc.json
+.next
+.settings
+.cache
+/.env
+/front/**/build
+/front/**/public/dist
+/node_modules
+_ide_helper*.php
+lefthook-local.yml
+logs
+public/dist
+test-ledger
+wip
+/tests/*.js
+*.env
+**/.env
+**/node_modules

indexer/chainweb-data/README.md

@@ -0,0 +1,40 @@
+## Kadena Chainweb Data
+
+Chainweb Data serves as a data source that provides real-time information about chains, transactions, and other elements of the Chainweb blockchain. By connecting the Chainweb Node to the Chainweb Data, our library allows the node to access real-time data to facilitate its functions, such as transaction execution, block validation, and updating the blockchain state.
+
+### Run
+```shell script
+$ docker compose up chainweb-data --build -d
+```
+
+### Vars
+The Chainweb-data container receives all database and node parameters via the .env file. Here are the variables:
+```
+# Chainweb-data vars
+CWD_NODE=""
+CWD_DB_PORT=""
+CWD_DB_USER=""
+CWD_DB_NAME=""
+CWD_DB_PASS=""
+CWD_DB_HOST=""
+```
+
+### Chainweb-data complex solution
+- server 
+- fill
+
+```shell script
+Node info: service-port=31351 --p2p-port=31350
+```
+
+### Health check
+```shell script
+docker inspect --format "{{json .State.Health }}" KadenaChainWebData | jq
+```
+
+### Endpoints
+- /txs/recent gets a list of recent transactions
+- /txs/search?search=foo&limit=20&offset=40 searches for transactions containing the string foo
+- /stats returns a few stats such as transaction count and coins in circulation
+- /coins returns just the coins in circulation
+

indexer/chainweb-data/docker-compose.yaml

@@ -0,0 +1,46 @@
+version: '3.9'
+
+# For usage instructions please take a look at the README.md file.
+networks:
+  indexer: {}
+
+volumes:
+  pg_data:
+
+services:
+  chainweb-data:
+    expose:
+      - "8888"
+    ports:
+      - "8888:8888"
+    build:
+      dockerfile: ./src/Dockerfile
+    env_file:
+      - .env
+    networks:
+      - indexer
+
+  postgresql:
+    image: postgres:14.5-bullseye
+    user: postgres
+    expose:
+      - "5432"
+    ports:
+      - "5432:5432"
+    env_file:
+      - .env
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+      - ./src/scripts/init-database.sh:/docker-entrypoint-initdb.d/init.sh
+    networks:
+      - indexer
+
+  # ########################################################################## #
+  # Debugging
+
+  debug:
+    image: alpine:latest
+    entrypoint:
+    - /bin/sh
+    profiles:
+    - debug

indexer/chainweb-data/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "indexer-cwd",
+  "version": "1.0.0",
+  "license": "UNLICENSED",
+  "private": true,
+  "scripts": {
+    "docker:stop:cwd": "docker compose stop chainweb-data",
+    "docker:up:pgl": "docker compose up postgresql --build -d",
+    "docker:up:cwd": "docker compose up chainweb-data --build -d",
+    "docker:logs:cwd": "docker compose logs chainweb-data --follow"
+  }
+}

indexer/chainweb-data/src/Dockerfile

@@ -0,0 +1,54 @@
+ARG UBUNTUVER=22.04
+FROM ubuntu:${UBUNTUVER}
+
+RUN apt-get update -y && apt-get upgrade -y \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y wget curl unzip dirmngr gnupg git cron lsof jq supervisor lsb-release \
+ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
+ && mkdir -p /usr/share/keyrings \
+ && mkdir -p /.gnupg \
+ && gpg --homedir /.gnupg --no-default-keyring --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 7FCC7D46ACCC4CF8
+
+RUN set -eux; \
+	groupadd -r postgres --gid=999; \
+	useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \
+	mkdir -p /var/lib/postgresql; \
+	chown -R postgres:postgres /var/lib/postgresql
+
+ENV PG_VERSION=15 \
+    FONTCONFIG_FILE=/etc/fonts/fonts.conf \
+    LANG=en_US.UTF-8 \
+    LC_ALL=en_US.UTF-8 \
+    LC_CTYPE=en_US.UTF-8 \
+    LOCALE_ARCHIVE=/usr/lib/locale/locale-archive \
+    UBUNTUVER=22.04
+
+RUN apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y locales postgresql-${PG_VERSION} postgresql-client-${PG_VERSION} postgresql-contrib-${PG_VERSION} \
+    && update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \
+    && locale-gen en_US.UTF-8 \
+    && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR "/usr/local/bin"
+
+RUN PACKAGE=$(curl --silent "https://api.github.com/repos/kadena-io/chainweb-data/releases/latest" | jq -r .assets[].browser_download_url | grep ${UBUNTUVER} ) \
+    && echo "Downloading file: ${PACKAGE}" \
+    && wget "${PACKAGE}" \
+    && unzip * \
+    && rm -rf *.zip \
+    && chmod +x chainweb-data
+
+RUN mkdir -p /var/log/supervisor
+
+COPY ./src/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY ./src/scripts/chainweb-data.sh /chainweb-data.sh
+COPY ./src/scripts/backfill.sh /backfill.sh
+COPY ./src/scripts/check-health.sh /check-health.sh
+
+RUN chmod 755 /backfill.sh
+RUN chmod 755 /chainweb-data.sh
+RUN chmod 755 /check-health.sh
+
+# HEALTHCHECK --start-period=10m --interval=1m --retries=5 --timeout=20s CMD /check-health.sh
+
+ENTRYPOINT ["/usr/bin/supervisord"]