From 10d31711fdd3207ef76197042a477807a8447ee0 Mon Sep 17 00:00:00 2001 From: caowei Date: Tue, 21 Feb 2023 15:47:47 +0800 Subject: [PATCH 1/5] add workflows for pr and pr pipeline --- .github/workflows/pipeline.yml | 86 ++++++++++++++++++++++++++++++++++ .github/workflows/pr.yml | 52 ++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 .github/workflows/pipeline.yml create mode 100644 .github/workflows/pr.yml diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..232a976 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,86 @@ +name: MXBENCH + +on: + push: + branches: [ master ] + +env: + pipeline_id: 17 + +jobs: + + start_pipeline: + runs-on: ubuntu-20.04 + container: + image: docker.pkg.github.com/ymatrix-data/matrixdb-ci/psqlci:latest + credentials: + username: ${{ secrets.CI_USER }} + password: ${{ secrets.PAT }} + steps: + - run: | + psql -c "DELETE FROM pipeline_status WHERE pipeline_id = ${{env.pipeline_id}} AND run_number = ${{github.run_number}} AND run_id = ${{github.run_id}}" + psql -c "INSERT INTO pipeline_status(pipeline_id, run_number, run_id, event_dt, status) VALUES(${{env.pipeline_id}}, ${{github.run_number}}, ${{github.run_id}}, now(), 'start')" + + build_and_test: + needs: [ start_pipeline ] + runs-on: ubuntu-20.04 + steps: + - name: Granting private modules access + run: | + git config --global url."https://${{secrets.CI_USER}}:${{secrets.PAT}}@github.com/ymatrix-data".insteadOf "https://github.com/ymatrix-data" + - uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version-file: go.mod + - name: golangci-lint + # https://github.com/golangci/golangci-lint-action + uses: golangci/golangci-lint-action@v3 + with: + # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. + version: latest + - name: Test + run: make test + - name: E2E + run: make e2e + - name: Build + run: make + + report_on_success: + if: ${{ success() }} + needs: [ build_and_test ] + runs-on: ubuntu-20.04 + container: + image: docker.pkg.github.com/ymatrix-data/matrixdb-ci/psqlci:latest + credentials: + username: ${{ secrets.CI_USER }} + password: ${{ secrets.PAT }} + steps: + - run: | + psql -c "INSERT INTO pipeline_status(pipeline_id, run_number, run_id, event_dt, status) VALUES(${{env.pipeline_id}}, ${{github.run_number}}, ${{github.run_id}}, now(), 'done')" + + report_on_cancel: + if: ${{ cancelled() }} + needs: [ build_and_test ] + runs-on: ubuntu-20.04 + container: + image: docker.pkg.github.com/ymatrix-data/matrixdb-ci/psqlci:latest + credentials: + username: ${{ secrets.CI_USER }} + password: ${{ secrets.PAT }} + steps: + - run: | + psql -c "INSERT INTO pipeline_status(pipeline_id, run_number, run_id, event_dt, status) VALUES(${{env.pipeline_id}}, ${{github.run_number}}, ${{github.run_id}}, now(), 'cancel')" + + report_on_failure: + if: ${{ failure() }} + needs: [ build_and_test ] + runs-on: ubuntu-20.04 + container: + image: docker.pkg.github.com/ymatrix-data/matrixdb-ci/psqlci:latest + credentials: + username: ${{ secrets.CI_USER }} + password: ${{ secrets.PAT }} + steps: + - run: | + psql -c "INSERT INTO pipeline_status(pipeline_id, run_number, run_id, event_dt, status) VALUES(${{env.pipeline_id}}, ${{github.run_number}}, ${{github.run_id}}, now(), 'error')" diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..eaf72ce --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,52 @@ +name: pr_pipeline + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + branches: [ master ] + +jobs: + + check_changes: + runs-on: ubuntu-20.04 + steps: + # https://github.com/marketplace/actions/pull-request-title-rules + - uses: dbhao/action-pr-title@master + id: wipcheck + with: + disallowed_prefixes: 'wip,rfc,do-not-merge,skipci,skip-ci,skip_ci,ciskip,ci-skip,ci_skip' # title should not start with the given prefix + prefix_case_sensitive: false # title prefix are case insensitive + - name: cancel_wip + if: steps.wipcheck.outputs.matched == 'true' + uses: andymckay/cancel-action@0.2 + + build_and_test: + needs: [ check_changes ] + runs-on: ubuntu-20.04 + steps: + - name: Granting private modules access + run: | + git config --global url."https://${{secrets.CI_USER}}:${{secrets.PAT}}@github.com/ymatrix-data".insteadOf "https://github.com/ymatrix-data" + + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: go.mod + + - name: golangci-lint + # https://github.com/golangci/golangci-lint-action + uses: golangci/golangci-lint-action@v3 + with: + # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. + version: latest + + - name: Test + run: make test + + - name: E2E + run: make e2e + + - name: Build + run: make From 842bbff0f217be546309089f25921ded303fade6 Mon Sep 17 00:00:00 2001 From: caowei Date: Tue, 21 Feb 2023 15:57:33 +0800 Subject: [PATCH 2/5] log accMiscTime2 to remove error for golint --- internal/engine/generator/telematics/telematics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/engine/generator/telematics/telematics.go b/internal/engine/generator/telematics/telematics.go index df24279..a3fc1ec 100644 --- a/internal/engine/generator/telematics/telematics.go +++ b/internal/engine/generator/telematics/telematics.go @@ -110,6 +110,7 @@ func (g *Generator) Close() error { log.Verbose("[Generator.TELEMATICS] Gen time: %s", time.Duration(accGenTime)) log.Verbose("[Generator.TELEMATICS] Write time: %s", time.Duration(accWriteTime)) log.Verbose("[Generator.TELEMATICS] Misc time1: %s", time.Duration(accMiscTime1)) + log.Verbose("[Generator.TELEMATICS] Misc time2: %s", time.Duration(accMiscTime2)) log.Verbose("[Generator.TELEMATICS] Write Cnt: %d", accWriteCnt) log.Verbose("[Generator.TELEMATICS] Acc Size: %d", accWriteSize) log.Verbose("[Generator.TELEMATICS] Max Size: %d", maxWriteSize) From 8517f9dce42d3ff0640d3682c60de9136c1c3f08 Mon Sep 17 00:00:00 2001 From: caowei Date: Tue, 21 Feb 2023 16:24:58 +0800 Subject: [PATCH 3/5] fix ginkgo --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index dfac3c3..3990001 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: build .PHONY: build linux_amd64 linux_arm64 clean lint race e2e test clean release -GINKGO:=go run github.com/onsi/ginkgo/ginkgo +GINKGO=$(GOPATH)/bin/ginkgo REPO_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) GIT_VERSION:=$(shell git tag --sort=-creatordate --points-at HEAD "${TAG_FILTER}*" | tail -n 1) @@ -21,6 +21,9 @@ MAIN_BRANCH_STR:=github.com/ymatrix-data/mxbench/internal/util.BranchStr=$(GIT_B MAIN_COMMIT_STR:=github.com/ymatrix-data/mxbench/internal/util.CommitStr=$(GIT_COMMIT) LDFLAGS:=-X $(MAIN_VERSION_STR) -X $(MAIN_BRANCH_STR) -X $(MAIN_COMMIT_STR) +$(GINKGO) : + $(GO_ENV) go install github.com/onsi/ginkgo/ginkgo + build: go build -o ./bin/mxbench -ldflags "$(LDFLAGS)" ./cmd/mxbench/main.go @@ -40,8 +43,8 @@ clean: lint: golangci-lint -v run -test: - $(GINKGO) -race ./internal/... +test: $(GINKGO) + ginkgo -race ./internal/... e2e: go build -tags e2e -ldflags "$(LDFLAGS)" -o mxbench_e2e ./cmd/mxbench From 84ed890526d401300b9e188e3de3682714780295 Mon Sep 17 00:00:00 2001 From: caowei Date: Tue, 21 Feb 2023 16:42:45 +0800 Subject: [PATCH 4/5] install ginkgo --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index a1f4203..48687a3 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -52,6 +53,7 @@ require ( golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect + golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/ini.v1 v1.66.2 // indirect diff --git a/go.sum b/go.sum index 0de7584..cedfcc0 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,7 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= @@ -275,6 +276,7 @@ golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From e7c9ef42799ee237c36b39f676e7cb479106ba64 Mon Sep 17 00:00:00 2001 From: caowei Date: Tue, 21 Feb 2023 16:48:37 +0800 Subject: [PATCH 5/5] Revert "fix ginkgo" This reverts commit 8517f9dce42d3ff0640d3682c60de9136c1c3f08. --- Makefile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 3990001..dfac3c3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: build .PHONY: build linux_amd64 linux_arm64 clean lint race e2e test clean release -GINKGO=$(GOPATH)/bin/ginkgo +GINKGO:=go run github.com/onsi/ginkgo/ginkgo REPO_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) GIT_VERSION:=$(shell git tag --sort=-creatordate --points-at HEAD "${TAG_FILTER}*" | tail -n 1) @@ -21,9 +21,6 @@ MAIN_BRANCH_STR:=github.com/ymatrix-data/mxbench/internal/util.BranchStr=$(GIT_B MAIN_COMMIT_STR:=github.com/ymatrix-data/mxbench/internal/util.CommitStr=$(GIT_COMMIT) LDFLAGS:=-X $(MAIN_VERSION_STR) -X $(MAIN_BRANCH_STR) -X $(MAIN_COMMIT_STR) -$(GINKGO) : - $(GO_ENV) go install github.com/onsi/ginkgo/ginkgo - build: go build -o ./bin/mxbench -ldflags "$(LDFLAGS)" ./cmd/mxbench/main.go @@ -43,8 +40,8 @@ clean: lint: golangci-lint -v run -test: $(GINKGO) - ginkgo -race ./internal/... +test: + $(GINKGO) -race ./internal/... e2e: go build -tags e2e -ldflags "$(LDFLAGS)" -o mxbench_e2e ./cmd/mxbench