fix(refactor-0.7): make CDK infra compatible to latest dockerfile

.dockerignore

@@ -0,0 +1,37 @@
+### Go template
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories
+vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+### CDK template
+# CDK asset staging directory.
+# For more information about AWS-CDK, see  https://docs.aws.amazon.com/cdk/
+.cdk.staging/
+**/cdk.out/
+
+### Misc
+.build
+.github
+.idea
+
+### Things that doesn't produce code or seed
+*.md
+scripts/temp_*
+scripts/produce_source_maps/all_tables.csv
+scripts/composed_streams.csv

.github/workflows/ci.yaml

@@ -48,24 +48,49 @@ jobs:
           go test ./internal/extensions/...
 
       - name: Docker Compose Up
+        # docker compose up --wait makes it exit with 1 even for a successful run
+        continue-on-error: true
         uses: isbang/compose-action@v1.5.1
         with:
           compose-file: "./compose.yaml"
+          # waits every container to be ready, so we're sure there's data already
+          up-flags: '--wait'
+        env:
+          # This is not a secret, it's hardcoded for the CI testing
+          PRIVATE_KEY: "0000000000000000000000000000000000000000000000000000000000000001"
+          WHITELIST_WALLETS: "0x304e893AdB2Ad8E8C37F4884Ad1EC3df8bA9bDcf"
 
-      - name: setup python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-      - name: Install Python dependencies
-        run: python -m pip install --upgrade pip pandas
+      # we need it to be manually checked, as above action will always exit with 1 even if it's successful
+      - name: Check compose status
+        run: |
+          output=$(docker ps -a --format "{{.Names}},{{.Image}},{{.Status}}")
+
+          function expect_status() {
+            image_name=$1
+            expected_status=$2
+            image_row=$(echo "$output" | grep "$image_name")
+            if echo "$image_row" | grep -q "$expected_status"; then
+              echo "$image_name exited with 0"
+            else
+              echo -e "$image_name isn't with expected status $expected_status\n"
+              echo -e "docker ps output: \n$output\n\n"
+
+              # extract the container name, knowing it's the first column
+              container_name=$(echo "$image_row" | cut -d ',' -f 1)
+
+              echo -e "docker logs for $image_name: \n"
+              docker logs $container_name | tail -n 500
+
+              exit 1
+            fi
+          }
+
+          # we ensure both containers are exited with 0
+          # if you are running locally, make sure any previous postgres volume state is clean
+          # otherwise, tsn-db will error for out for block state being out of sync.
+          expect_status "tsn-db:local" "Up "
+          expect_status "push-tsn-data:local" "Exited (0)"
 
-      # It takes time to download the kwil binary and extract kwil-cli so I push kwil-cli to the repo.
-      # It needed to seed database. Are there any better way to do this?
-      # PRIVATE_KEY and WHITELIST_WALLETS are hardcoded for testing reasons
-      - name: Seed Database and Run Tests
+      - name: Run Tests
         run: |
-          timeout 300 docker logs tsn-db-tsn-db-1 -f | grep  -m1 -oP '"Health check state changed","state":"up"' || { ec=$?; [ $ec -eq 141 ] && true || (exit $ec); }
-          export PRIVATE_KEY="0000000000000000000000000000000000000000000000000000000000000001"
-          export WHITELIST_WALLETS="0x304e893AdB2Ad8E8C37F4884Ad1EC3df8bA9bDcf"
-          scripts/setup.sh
           scripts/ci-tests.sh

.gitignore

@@ -1,9 +1,13 @@
 .git
-.env
+.env*
+!.env.example
 .idea
 
 .build/kwild
 
+go.work*
+vendor
+
 scripts/temp_csv
 scripts/composed_streams.csv
 scripts/temp_composed_schemas

README.md

@@ -48,26 +48,15 @@ For more information on running nodes, and how to run a multi-node network, refe
 
 ### Resetting local deployments
 
-By default, `kwild` stores all data in `~/.kwild`. To reset the data on a deployment, remove the data directory while the node is stopped:
+You can clear the local Kwil data by running the following command:
 
 ```shell
-rm -r ~/.kwild
+task clear-data
 ```
 
-## Docker Compose Deployment
-
-### Run TSN-DB with Postgres using Docker Compose
-
-To run the TSN-DB with Postgres using Docker Compose, run the following command:
-```shell
-task compose
-```
-
-### Seed Data
+### Configure the kwil-cli
 
-#### Configure the kwil-cli
-
-To seed data into the TSN-DB, you will need to configure the kwil-cli.
+To interact with the the TSN-DB, you will need to configure the kwil-cli.
 ```shell
 kwil-cli configure
 
@@ -78,10 +67,19 @@ Private Key: <any ethereum private key>
 # use private key 0000000000000000000000000000000000000000000000000000000000000001 for testing
 ```
 
+## Docker Compose Deployment
+
+### Run TSN-DB with Postgres using Docker Compose
+
+To run the TSN-DB with Postgres using Docker Compose, run the following command:
+```shell
+task compose
+```
 
+This will start the TSN-DB and Postgres in Docker containers, which is already seeded.
 
 #### Seed Data
-To seed data into the TSN-DB, run the following command:
+If you need to manually seed data into the TSN-DB, run the following command:
 ```shell
 task seed
 ```

Taskfile.yml

@@ -54,6 +54,11 @@ tasks:
         && mv ./kwil-cli .build
         && rm ./kwil-db.tar.gz
 
+  clear-data:
+    cmds:
+      - rm -r ~/.kwild || true
+      - docker volume rm tsn-db_data-kwil-postgres tsn-db_data-push-tsn-data -f
+
   seed:
     desc: Seed the database
     cmds:

compose.yaml

@@ -10,8 +10,20 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
+    # persis data
+    volumes:
+      - type: volume
+        source: data-kwil-postgres
+        target: /var/lib/postgresql/data
+        volume:
+          nocopy: true
+    networks:
+      - tsn-network
 
   tsn-db:
+    container_name: tsn-db
+    hostname: tsn-db
+    image: "tsn-db:local"
     build:
       context: .
       dockerfile: ./deployments/Dockerfile
@@ -20,3 +32,44 @@ services:
     depends_on:
       kwil-postgres:
         condition: service_healthy
+    networks:
+      - tsn-network
+
+  push-tsn-data:
+    image: "push-tsn-data:local"
+    build:
+      context: .
+      dockerfile: ./deployments/push-tsn-data.dockerfile
+    volumes:
+      - type: volume
+        source: data-push-tsn-data
+        # should only run once at deployment
+        target: /firstrun
+        volume:
+          nocopy: false
+    depends_on:
+      tsn-db:
+        condition: service_started
+    networks:
+      - tsn-network
+    environment:
+      # should come from environment
+      - PRIVATE_KEY=${PRIVATE_KEY:?PRIVATE_KEY is necessary for deployment}
+      # this is not required, but if it is set, it should be a comma separated list of wallet addresses (e.g. 0x1234,0x5678)
+      # if not set, only the owner of the private key will be able to query
+      - WHITELIST_WALLETS=${WHITELIST_WALLETS}
+
+  finisher:
+    container_name: wait-for-completion
+    image: hello-world
+    depends_on:
+      push-tsn-data:
+        condition: service_completed_successfully
+
+networks:
+  tsn-network:
+    driver: bridge
+
+volumes:
+  data-kwil-postgres:
+  data-push-tsn-data:

deployments/infra/.gitignore

@@ -0,0 +1,21 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+cdk.iml
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# go.sum should be committed
+!go.sum
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

deployments/infra/README.md

@@ -0,0 +1,12 @@
+# Welcome to your CDK Go project!
+
+This is a blank project for CDK development with Go.
+
+The `cdk.json` file tells the CDK toolkit how to execute your app.
+
+## Useful commands
+
+ * `cdk deploy`      deploy this stack to your default AWS account/region
+ * `cdk diff`        compare deployed stack with current state
+ * `cdk synth`       emits the synthesized CloudFormation template
+ * `go test`         run unit tests

deployments/infra/buildx.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# covert docker build to docker buildx build --load
+if [[ "$1" == "build" ]]; then
+  docker buildx build --load "${@:2}"
+else
+  docker "$@"
+fi

deployments/infra/cdk.context.json

@@ -0,0 +1,42 @@
+{
+  "availability-zones:account=344375646931:region=us-east-1": [
+    "us-east-1a",
+    "us-east-1b",
+    "us-east-1c",
+    "us-east-1d",
+    "us-east-1e",
+    "us-east-1f"
+  ],
+  "vpc-provider:account=344375646931:filter.isDefault=true:region=us-east-2:returnAsymmetricSubnets=true": {
+    "vpcId": "vpc-031fd5cc4065f5693",
+    "vpcCidrBlock": "172.31.0.0/16",
+    "ownerAccountId": "344375646931",
+    "availabilityZones": [],
+    "subnetGroups": [
+      {
+        "name": "Public",
+        "type": "Public",
+        "subnets": [
+          {
+            "subnetId": "subnet-02a0a112084190fa9",
+            "cidr": "172.31.0.0/20",
+            "availabilityZone": "us-east-2a",
+            "routeTableId": "rtb-06027a22f2823df2a"
+          },
+          {
+            "subnetId": "subnet-00182b35be3ac1e96",
+            "cidr": "172.31.16.0/20",
+            "availabilityZone": "us-east-2b",
+            "routeTableId": "rtb-06027a22f2823df2a"
+          },
+          {
+            "subnetId": "subnet-030496ef398fbf3ce",
+            "cidr": "172.31.32.0/20",
+            "availabilityZone": "us-east-2c",
+            "routeTableId": "rtb-06027a22f2823df2a"
+          }
+        ]
+      }
+    ]
+  }
+}