refactor: sync repo with kwil v0.7

.dockerignore

@@ -1,30 +1,37 @@
-.git/
-.build/
-.github/
-.vscode/
-.idea/
-.wal/
-build/package/docker/
-.gitattributes
-.gitignore
-.DS_Store
-test/
-proto/
-deployments/
-.editorconfig
-.gitmodules
-.golangci.yml
-.goreleaser.yaml
-.local_env
-cmd/kwild/kwild
-cmd/kwil-admin/kwil-admin
-cmd/kwil-cli/kwil-cli
-**/__debug_bin.exe
-**/__debug_bin*
-**/_*
-**/.testnet
-truflation/infra
+### Go template
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
 
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories
+vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+### CDK template
+# CDK asset staging directory.
+# For more information about AWS-CDK, see  https://docs.aws.amazon.com/cdk/
+.cdk.staging/
+**/cdk.out/
+
+### Misc
+.build
+.github
+.idea
+
+### Things that doesn't produce code or seed
 *.md
-truflation/composed_streams.csv
-truflation/scripts/temp_*/**/*
+scripts/temp_*
+scripts/produce_source_maps/all_tables.csv
+scripts/composed_streams.csv

.github/workflows/ci.yaml

@@ -8,13 +8,13 @@ on:
   push:
     branches:
       - main
+      - refactor/upgrade-zero-seven # TODO: remove this line after the refactor is done
 
 jobs:
   acceptance-test:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }} # only run on non-draft PRs
+    if: ${{ !github.event.pull_request.draft }}
     steps:
-      # Clear cache to avoid no space left on device
       - name: Clear cache
         if: ${{ !env.ACT }} # skip during local actions testing
         run: |
@@ -23,112 +23,68 @@ jobs:
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v2
-        with:
-          version: '23.4'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Install Taskfile
-        uses: arduino/setup-task@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
 
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
           go-version: '1.21'
-          check-latest: true
-
-      - name: Install dependencies
-        env:
-          GH_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          go version
-          git config --global url."https://${GH_ACCESS_TOKEN}:x-oauth-basic@github.com/truflation/".insteadOf "https://github.com/truflation/"
-          task install:deps
-          go mod download
 
       - name: Lint
+        if: ${{ !env.ACT }} # skip during local actions testing
         uses: golangci/golangci-lint-action@v4
         with:
-          skip-pkg-cache: true
+          args: --timeout=30m --issues-exit-code=0
+          skip-cache: true
 
       - name: Run Go Tests
         run: |
-          task test:unit
-
-      - name: Generate go vendor
-        #for faster builds and private repos, need to run this after pb:compile:v1
-        run: |
-          task vendor
+          go test ./internal/...
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Restore Docker layers
-        id: restore_cache
-        uses: actions/cache/restore@v4
-        with:
-          path: /tmp/.buildx-cache-tsn-db
-          # we don't use sha, so we can reuse between commits.
-          # docker takes care of invalidating the cache if the build context changes
-          #key: ${{ runner.os }}-buildx-tsn-db-${{ github.sha }}
-          key: ${{ runner.os }}-buildx-tsn-db
-
-      - name: Build image
-        id: docker_build_tsn_db
-        uses: docker/build-push-action@v4
+      - name: Docker Compose Up
+        # docker compose up --wait makes it exit with 1 even for a successful run
+        continue-on-error: true
+        uses: isbang/compose-action@v1.5.1
         with:
-          context: .
-          builder: ${{ steps.buildx.outputs.name }}
-          build-args: |
-            git_commit=${{ github.sha }}
-            version=${{ steps.config.outputs.version }}
-            build_time=${{ github.event.release.created_at }}
-            # hardcoded for testing reasons
-            whitelist_wallets=0x304e893AdB2Ad8E8C37F4884Ad1EC3df8bA9bDcf
-          file: ./truflation/docker/tsn.dockerfile
-          cache-from: type=local,src=/tmp/.buildx-cache-tsn-db
-          cache-to: type=local,dest=/tmp/.buildx-cache-tsn-db-new
-          tags: tsn-db:latest
-          load: true
-
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache-tsn-db
-          mv /tmp/.buildx-cache-tsn-db-new /tmp/.buildx-cache-tsn-db
-
-      # we save already to be able to free up space
-      - name: Save Docker layers Cache
-        uses: actions/cache/save@v4
-        with:
-          path: /tmp/.buildx-cache-tsn-db
-          # we don't use sha, so we can reuse between commits.
-          # docker takes care of invalidating the cache if the build context changes
-          #key: ${{ runner.os }}-buildx-tsn-db-${{ github.sha }}
-          key: ${{ steps.restore_cache.outputs.cache-primary-key }}
+          compose-file: "./compose.yaml"
+          # waits every container to be ready, so we're sure there's data already
+          up-flags: '--wait'
+        env:
+          # This is not a secret, it's hardcoded for the CI testing
+          PRIVATE_KEY: "0000000000000000000000000000000000000000000000000000000000000001"
+          WHITELIST_WALLETS: "0x304e893AdB2Ad8E8C37F4884Ad1EC3df8bA9bDcf"
 
-      - name: Free Up Space
+      # we need it to be manually checked, as above action will always exit with 1 even if it's successful
+      - name: Check compose status
         run: |
-          docker system prune -f
-          rm -rf /tmp/.buildx-cache-tsn-db
-
-      - name: Run tsn-db image
+          output=$(docker ps -a --format "{{.Names}},{{.Image}},{{.Status}}")
+
+          function expect_status() {
+            image_name=$1
+            expected_status=$2
+            image_row=$(echo "$output" | grep "$image_name")
+            if echo "$image_row" | grep -q "$expected_status"; then
+              echo "$image_name exited with 0"
+            else
+              echo -e "$image_name isn't with expected status $expected_status\n"
+              echo -e "docker ps output: \n$output\n\n"
+
+              # extract the container name, knowing it's the first column
+              container_name=$(echo "$image_row" | cut -d ',' -f 1)
+
+              echo -e "docker logs for $image_name: \n"
+              docker logs $container_name | tail -n 500
+
+              exit 1
+            fi
+          }
+
+          # we ensure both containers are exited with 0
+          # if you are running locally, make sure any previous postgres volume state is clean
+          # otherwise, tsn-db will error for out for block state being out of sync.
+          expect_status "tsn-db:local" "Up "
+          expect_status "push-tsn-data:local" "Exited (0)"
+
+      - name: Run Tests
         run: |
-          task build:cli
-          docker_id=$(docker run -p 8080:8080 -d tsn-db:latest)
-
-          # wait for the server to start
-          # -f = follow logs | -m1 = only one match | -o = only matching | -P = perl regex
-          # last pipe avoid failing on broken pipes (141), preserving other error codes
-          # PS: multiline command with \ isn't working here
-          timeout 300 docker logs $docker_id -f | grep  -m1 -oP '"Health check state changed","state":"up"' || { ec=$?; [ $ec -eq 141 ] && true || (exit $ec); }
-
-          ./truflation/scripts/ci-tests.sh || (echo "Command failed, printing logs:" && docker logs $docker_id && exit 1)
+          scripts/ci-tests.sh

.github/workflows/deploy-staging.yaml

@@ -14,25 +14,12 @@ jobs:
     runs-on: ubuntu-latest
 #    needs: [ tests ]  # require tests to pass before deploy runs
     steps:
-      - uses: waycarbon/github-action-container-cleanup@v1
-
       - name: Checkout
         uses: actions/checkout@v3
         with:
           submodules: true
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v2
-        with:
-          version: '23.4'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Install Taskfile
-        uses: arduino/setup-task@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
@@ -44,11 +31,8 @@ jobs:
         env:
           GH_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          go version
-          git config --global url."https://${GH_ACCESS_TOKEN}:x-oauth-basic@github.com/truflation/".insteadOf "https://github.com/truflation/"
-          task install:deps
           go work init
-          go work use truflation/infra
+          go work use deployments/infra
           go mod download
           go work vendor
 
@@ -57,12 +41,6 @@ jobs:
 #        run: |
 #          go work vendor
 
-      - name: configure variables
-        id: config
-        run: |
-          version=`echo ${{ github.event.release.tag_name }} | sed 's/^v//'`
-          echo "version=$version" >> $GITHUB_OUTPUT
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
@@ -98,18 +76,16 @@ jobs:
       # this step includes building the docker image. See main_cdk.go for details on args used
       - name: Deploy
         run: |
-          cd truflation/infra
+          cd deployments/infra
           # this must be on absolute path
           export CDK_DOCKER=$(pwd)/buildx.sh
 
           cdk deploy --require-approval never
         env:
-          GIT_COMMIT: ${{ github.sha }}
-          GIT_VERSION: ${{ steps.config.outputs.version }}
-          BUILD_TIME: ${{ github.event.release.created_at }}
+          PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
           WHITELIST_WALLETS: ${{ vars.whitelist_wallets }}
 
       - name: Move cache
         run: |
           rm -rf /tmp/.buildx-cache-tsn-db
-          mv /tmp/.buildx-cache-tsn-db-new /tmp/.buildx-cache-tsn-db
+          mv /tmp/.buildx-cache-tsn-db-new /tmp/.buildx-cache-tsn-db

.github/workflows/release.yaml

@@ -122,4 +122,4 @@ jobs:
 
       - name: deploy by release type to different eks cluster
         run: |
-          echo "will deploy ${{ steps.config.outputs.version }} to eks ${{ steps.config.outputs.cluster }} cluster"
+          echo "will deploy ${{ steps.config.outputs.version }} to eks ${{ steps.config.outputs.cluster }} cluster"