New build system (#210)

* New build system

* fixes

* fix

* fix

* fix

* fix generation

* fix generation

* fix generation

* fix generation

* clean up the makefile

* update coreum builder

* fix missing profile

* fix

* fix

* Merge master into wojtek/ci

* support `bin` in `PATH`

* fix

* fix

* linter

* Use self-hosted runner

* Update crust

* fix

* replace gocheck

* replace gocheck

* fix

.github/workflows/contract-ci.yml

@@ -9,19 +9,9 @@ env:
   CARGO_TERM_COLOR: always
 jobs:
   ci:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - uses: actions/checkout@v4
-      - name: Set up cargo/wasm cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            contract/target/
-          key: ${{ runner.os }}-cargo-wasm-cache-${{ hashFiles('**/Cargo.lock') }}
       - name: Lint contract
         run: make lint-contract
       - name: Build contract

.github/workflows/relayer-ci.yml

@@ -20,129 +20,25 @@ jobs:
         ]
         include:
           - ci_step: "lint"
-            command: "coreumbridge-xrpl-builder lint"
-            wasm-cache: false
-            linter-cache: true
-            docker-cache: false
+            command: make lint
           - ci_step: "test"
-            command: "coreumbridge-xrpl-builder test fuzz-test"
-            wasm-cache: false
-            linter-cache: false
-            docker-cache: false
+            command: make test fuzz-test
           - ci_step: "integration tests contract"
-            command: |
-              coreumbridge-xrpl-builder build/contract images
-              coreum-builder build images
-              crust znet start --profiles=1cored,xrpl --timeout-commit 0.5s
-              coreumbridge-xrpl-builder integration-tests/contract
-            wasm-cache: true
-            linter-cache: false
-            docker-cache: true
+            command: make integration-tests-contract
           - ci_step: "integration tests processes"
-            command: |
-              coreumbridge-xrpl-builder build/contract images
-              coreum-builder build images
-              crust znet start --profiles=1cored,xrpl --timeout-commit 0.5s
-              coreumbridge-xrpl-builder integration-tests/processes
-            wasm-cache: true
-            linter-cache: false
-            docker-cache: true
+            command: make integration-tests-processes
           - ci_step: "integration tests xrpl"
-            command: |
-              coreumbridge-xrpl-builder build/contract images
-              coreum-builder build images
-              crust znet start --profiles=1cored,xrpl --timeout-commit 0.5s
-              coreumbridge-xrpl-builder integration-tests/xrpl
-            wasm-cache: true
-            linter-cache: false
-            docker-cache: true
+            command: make integration-tests-xrpl
 
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     steps:
       - name: Go version used to build builder
         run: go version
       - name: Checkout coreumbridge-xrpl
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          path: coreumbridge-xrpl
-      - name: Checkout crust
-        uses: actions/checkout@v4
-        with:
-          repository: CoreumFoundation/crust
-          path: crust
-          ref: 66e0a14ecac37fe053780b6d814231bd98eb2240
-      - name: Checkout coreum
-        uses: actions/checkout@v4
-        with:
-          repository: CoreumFoundation/coreum
-          path: coreum
-          ref: 4e06879add40dc0a64c48a87f4b293c08b88f4d9
-      - name: Set up build system
-        run: |
-          echo "$(pwd)/coreumbridge-xrpl/bin" >> $GITHUB_PATH
-          echo "$(pwd)/coreum/bin" >> $GITHUB_PATH
-          echo "$(pwd)/crust/bin" >> $GITHUB_PATH
-          coreumbridge-xrpl/bin/coreumbridge-xrpl-builder build/me
-      - name: Retrieve go version
-        id: goversion
-        run: echo "GO_VERSION=$(coreumbridge-xrpl/bin/go version)" >> $GITHUB_OUTPUT
-      - name: Print go version
-        run: echo ${{ steps.goversion.outputs.GO_VERSION }}
-      - name: Setup go cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ steps.goversion.outputs.GO_VERSION }}
-      - name: Setup WASM cache
-        uses: actions/cache@v3
-        if: ${{ matrix.wasm-cache }}
-        with:
-          # we need to exclude code-hashes.json from the rest, since the invalidation condition for it is not
-          # same as WASM cache, but it is same as smart contract artifacts.
-          path: |
-            ~/.cache/crust/wasm
-            !/.cache/crust/wasm/code-hashes.json
-          key: ${{ runner.os }}-wasm-cache-${{ hashFiles('~/.cache/crust/wasm/**/*.rs') }}
-      - name: Setup linter cache
-        uses: actions/cache@v3
-        if: ${{ matrix.linter-cache }}
-        with:
-          path: ~/.cache/golangci-lint
-          key: ${{ runner.os }}-linter-cache-2-${{ steps.goversion.outputs.GO_VERSION }}
-      - name: Get Date
-        id: get-year-week
-        run: |
-          echo "date=$(/bin/date -u "+%Y-%U")" >> $GITHUB_OUTPUT
-        shell: bash
-      - name: Set docker cache
-        uses: ScribeMD/docker-cache@0.5.0
-        if: ${{ matrix.docker-cache }}
-        with:
-          key: ${{ runner.os }}-docker-v3-scribemd-${{ steps.get-year-week.outputs.date }} # year-week key
-      - name: Setup smart contract build cache
-        uses: actions/cache@v3
-        with:
-          # we need to cache code-hashes.json under the same key as the artifacts, since the invalidation condition
-          # for both of them are the same.
-          path: |
-            ${{ github.workspace }}/coreumbridge-xrpl/contract/artifacts/* 
-            ~/.cache/crust/wasm/code-hashes.json
-          key: ${{ runner.os }}-cache-smart-contracts-${{ hashFiles('./coreumbridge-xrpl/contract/**/*.rs') }}
-        if: ${{ matrix.wasm-cache }}
-      - name: Prepare dependencies
-        # FIXME (wojciech): Execute `download` commands for crust and coreum builders once available
-        run: |
-          crust build/me
-          crust build
-          crust znet remove
-          coreum-builder build/me
-          coreumbridge-xrpl-builder build/me
-          coreumbridge-xrpl-builder download
       - name: Run ${{ matrix.ci_step }}
-        working-directory: ./coreumbridge-xrpl
         run: ${{ matrix.command }}
       - name: Dump docker logs on failure
         if: failure()

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   release:
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     steps:
       - name: Checkout the repo 
         uses: actions/checkout@v4

Makefile

@@ -1,159 +1,78 @@
-GO_IMPORT_PREFIX=github.com/CoreumFoundation
-GO_SCAN_FILES := $(shell find . -type f -name '*.go' -not -name '*mock.go' -not -name '*_gen.go' -not -path "*/vendor/*")
+BUILDER = ./bin/coreumbridge-xrpl-builder
+
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 CONTRACT_DIR:=$(ROOT_DIR)/contract
-INTEGRATION_TESTS_DIR:=$(ROOT_DIR)/integration-tests
-RELAYER_DIR:=$(ROOT_DIR)/relayer
-BUILD_DIR?=$(ROOT_DIR)/build
-GIT_TAG:=$(shell git describe --tags --exact-match 2>/dev/null)
-ifeq ($(GIT_TAG),)
-GIT_TAG:=devel
-endif
-GIT_SHA:=$(shell git rev-parse HEAD)
-DOCKER_PUSH_TAG?=$(shell git describe --tags --exact-match 2>/dev/null || git rev-parse HEAD)
-LD_FLAGS:="-extldflags=-static \
--X 'github.com/CoreumFoundation/coreumbridge-xrpl/relayer/buildinfo.VersionTag=$(GIT_TAG)' \
--X 'github.com/CoreumFoundation/coreumbridge-xrpl/relayer/buildinfo.GitCommit=$(GIT_SHA)' \
-"
-GOOS?=
-GOARCH?=
-BINARY_NAME?=coreumbridge-xrpl-relayer
-RELEASE_VERSION=v1.1.0
-
-###############################################################################
-###                                  Build                                  ###
-###############################################################################
+
+.PHONY: znet-start
+znet-start:
+	$(BUILDER) znet start --profiles=bridge-xrpl
+
+.PHONY: znet-remove
+znet-remove:
+	$(BUILDER) znet remove
+
+.PHONY: lint
+lint:
+	$(BUILDER) lint
+
+.PHONY: test
+test:
+	$(BUILDER) test
+
+.PHONY: fuzz-test
+fuzz-test:
+	$(BUILDER) fuzz-test
 
 .PHONY: build-relayer
 build-relayer:
-	cd $(RELAYER_DIR) && CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build --trimpath -mod=readonly -ldflags $(LD_FLAGS)  -o $(BUILD_DIR)/$(BINARY_NAME) ./cmd
-
-.PHONY: release-relayer
-release-relayer:
-	@$(MAKE) build-relayer-in-docker GOOS=linux GOARCH=amd64 BINARY_NAME=relayer-linux-amd64
-	@$(MAKE) build-relayer-in-docker GOOS=linux GOARCH=arm64 BINARY_NAME=relayer-linux-arm64
-	@$(MAKE) build-relayer-in-docker GOOS=darwin GOARCH=amd64 BINARY_NAME=relayer-darwin-amd64
-	@$(MAKE) build-relayer-in-docker GOOS=darwin GOARCH=arm64 BINARY_NAME=relayer-darwin-arm64
-
-.PHONY: build-relayer-docker
-build-relayer-docker:
-	docker buildx build --build-arg GOOS=$(GOOS) --build-arg GOARCH=$(GOARCH) -f $(RELAYER_DIR)/Dockerfile . -t coreumbridge-xrpl-relayer:local
-
-.PHONY: push-relayer-docker
-push-relayer-docker: build-relayer-docker
-	docker image tag coreumbridge-xrpl-relayer:local coreumfoundation/coreumbridge-xrpl-relayer:$(DOCKER_PUSH_TAG)
-	docker image push coreumfoundation/coreumbridge-xrpl-relayer:$(DOCKER_PUSH_TAG)
-
-.PHONY: build-relayer-in-docker
-build-relayer-in-docker:
-	make build-relayer-docker
-	mkdir -p $(BUILD_DIR)
-	docker run --rm --entrypoint cat coreumbridge-xrpl-relayer:local /bin/coreumbridge-xrpl-relayer > $(BUILD_DIR)/$(BINARY_NAME)
+	$(BUILDER) build/relayer
 
 .PHONY: build-contract
 build-contract:
-	docker run --user $(id -u):$(id -g) --rm -v $(CONTRACT_DIR):/code \
-      -v $(CONTRACT_DIR)/target:/target \
-      -v $(CONTRACT_DIR)/target:/usr/local/cargo/registry \
-      cosmwasm/optimizer:0.15.0
-
-.PHONY: build-dev-contract
-build-dev-contract:
-	rustup target add wasm32-unknown-unknown
-	cargo install wasm-opt --locked
-	# Those RUSTFLAGS reduce binary size considerably
-	cd $(CONTRACT_DIR) && RUSTFLAGS='-C link-arg=-s' cargo wasm
-	mkdir -p $(BUILD_DIR)
-	cp $(CONTRACT_DIR)/target/wasm32-unknown-unknown/release/coreumbridge_xrpl.wasm $(BUILD_DIR)/coreumbridge_xrpl_not_opt.wasm
-	wasm-opt -Os --signext-lowering $(BUILD_DIR)/coreumbridge_xrpl_not_opt.wasm -o $(BUILD_DIR)/coreumbridge_xrpl.wasm
-	rm $(BUILD_DIR)/coreumbridge_xrpl_not_opt.wasm
-
-###############################################################################
-###                               Development                               ###
-###############################################################################
-
-.PHONY: fmt-go
-fmt-go:
-	which gofumpt || go install mvdan.cc/gofumpt@v0.5.0
-	which gogroup || go install github.com/vasi-stripe/gogroup/cmd/gogroup@v0.0.0-20200806161525-b5d7f67a97b5
-	gofumpt -lang=v2.1 -extra -w $(GO_SCAN_FILES)
-	gogroup -order std,other,prefix=$(GO_IMPORT_PREFIX) -rewrite $(GO_SCAN_FILES)
-
-.PHONY: mockgen-go
-mockgen-go:
-	which mockgen || go install github.com/golang/mock/mockgen@v1.6.0
-	cd $(RELAYER_DIR) && go generate ./...
-	make fmt-go
-
-.PHONY: lint-go
-lint-go:
-	crust lint/current-dir
+	$(BUILDER) build/contract
 
-.PHONY: lint-contract
-lint-contract:
-	cd $(CONTRACT_DIR) && cargo clippy --verbose -- -D warnings || exit 1;
+.PHONY: images
+images:
+	$(BUILDER) images
 
-.PHONY: test-integration
-test-integration:
-	# test each directory separately to prevent faucet concurrent access
-	for d in $(INTEGRATION_TESTS_DIR)/*/; \
-	 do make test-single-integration TESTS_DIR="$$d" || exit 1; \
-	done
+.PHONY: release
+release:
+	$(BUILDER) release
 
-.PHONY: test-single-integration
-test-single-integration:
-	cd $(TESTS_DIR) && go test -v --tags=integrationtests -mod=readonly -parallel=20 -timeout 10m ./... || exit 1;
+.PHONY: release-images
+release-images:
+	$(BUILDER) release/images
 
-.PHONY: test-relayer
-test-relayer:
-	cd $(RELAYER_DIR) && go clean -testcache && go test -v -mod=readonly -parallel=20 -timeout 30m -race ./...
+.PHONY: integration-tests-xrpl
+integration-tests-xrpl:
+	$(BUILDER) integration-tests/xrpl
 
-.PHONY: test-contract
-test-contract:
-	cd $(CONTRACT_DIR) && cargo test --verbose
+.PHONY: integration-tests-processes
+integration-tests-processes:
+	$(BUILDER) integration-tests/processes
+
+.PHONY: integration-tests-contract
+integration-tests-contract:
+	$(BUILDER) integration-tests/contract
 
-.PHONY: test-fuzz
-test-fuzz:
-	cd $(RELAYER_DIR)/processes && go test -v -mod=readonly -run ^FuzzAmountConversionCoreumToXRPLAndBack$$  -fuzz ^FuzzAmountConversionCoreumToXRPLAndBack$$ -fuzztime 20s
-	cd $(RELAYER_DIR)/processes && go test -v -mod=readonly -run ^FuzzAmountConversionCoreumToXRPLAndBack_ExceedingSignificantNumber$$  -fuzz  ^FuzzAmountConversionCoreumToXRPLAndBack_ExceedingSignificantNumber$$ -fuzztime 20s
+.PHONY: integration-tests-stress
+integration-tests-stress:
+	$(BUILDER) integration-tests/stress
 
-.PHONY: restart-dev-env
-restart-dev-env:
-	crust znet remove && crust znet start --profiles=1cored,xrpl --timeout-commit 0.3s
+# FIXME (wojtek): Builder does not support the following actions
 
-.PHONY: build-dev-env
-build-dev-env:
-	crust build
-	crust images
-	coreum-builder images
+.PHONY: lint-contract
+lint-contract:
+	cd $(CONTRACT_DIR) && cargo clippy --verbose -- -D warnings || exit 1;
 
-.PHONY: build-bridge-znet-env
-build-bridge-znet-env:
-	make build-dev-env
-	make build-relayer-docker
+.PHONY: test-contract
+test-contract:
+	cd $(CONTRACT_DIR) && cargo test --verbose
 
 .PHONY: restart-bridge-znet-env
 restart-bridge-znet-env:
 	docker compose -p bridge-znet -f ./infra/composer/docker-compose.yaml stop
-	crust znet remove
+	$(BUILDER) znet remove
 	docker compose -p bridge-znet -f ./infra/composer/docker-compose.yaml down
-	crust znet start --profiles=bridge-xrpl
+	$(BUILDER)znet start --profiles=bridge-xrpl
 	docker compose -p bridge-znet -f ./infra/composer/docker-compose.yaml up -d
-
-.PHONY: download-released-contract
-download-released-contract:
-	mkdir -p $(BUILD_DIR)
-	curl --fail -LJ -o $(BUILD_DIR)/coreumbridge_xrpl_$(RELEASE_VERSION).wasm https://github.com/CoreumFoundation/coreumbridge-xrpl/releases/download/$(RELEASE_VERSION)/coreumbridge_xrpl.wasm
-
-.PHONY: smoke
-smoke:
-	make lint-contract
-	make build-dev-contract
-	make test-contract
-	make mockgen-go
-	make fmt-go
-	make test-relayer
-	make restart-dev-env
-	make test-integration
-	# last since checks the committed files
-	make lint-go