From 9b97f878988cafb5e51a466e8374312e86b05c3d Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 11:54:54 -0700 Subject: [PATCH 01/16] Do deploy at the first CI stage --- .travis.yml | 136 ++++++++++++++++---------------- scripts/travis/deploy_client.sh | 13 +-- 2 files changed, 75 insertions(+), 74 deletions(-) diff --git a/.travis.yml b/.travis.yml index fa75f54bc8..23b5e4c7bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,75 +34,72 @@ install: - echo "skip install" jobs: include: - - stage: BuildAndTest - script: - - set -e - - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - - docker run --rm -it -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:dev - pre-commit run -a --show-diff-on-failure - - env: SQLFLOW_TEST_DB=mysql - script: - - set -e - - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci - scripts/test/units.sh - - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci - scripts/upload_coveralls.sh - - env: SQLFLOW_TEST_DB=hive # run more parallel tests in the same stage: - script: - - set -e - - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - - docker pull sqlflow/gohive:dev - - docker run -d --name=hive sqlflow/gohive:dev python3 -m http.server 8899 - - docker run --rm -it --net=container:hive - -v $TRAVIS_BUILD_DIR:/work -w /work - -v $GOPATH:/root/go - sqlflow:ci scripts/test/hive.sh - - env: SQLFLOW_TEST_DB=maxcompute - script: - - set -e - - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh - - docker run --rm --net=host - -v /var/run/docker.sock:/var/run/docker.sock - -v $HOME/.kube:/root/.kube - -v /home/$USER/.minikube/:/home/$USER/.minikube/ - -v $TRAVIS_BUILD_DIR:/work -w /work - sqlflow:ci scripts/test/ipython.sh - - docker run --rm --net=host - -e SQLFLOW_TEST_DB_MAXCOMPUTE_AK=$MAXCOMPUTE_AK - -e SQLFLOW_TEST_DB_MAXCOMPUTE_SK=$MAXCOMPUTE_SK - -v /var/run/docker.sock:/var/run/docker.sock - -v $HOME/.kube:/root/.kube - -v /home/$USER/.minikube/:/home/$USER/.minikube/ - -v $TRAVIS_BUILD_DIR:/work -w /work - sqlflow:ci scripts/test/maxcompute.sh - - env: SQLFLOW_TEST=java - script: - - set -e - - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - - docker run --rm - -v $TRAVIS_BUILD_DIR:/work -w /work - sqlflow:ci scripts/test/java.sh - - env: SQLFLOW_TEST=workflow - script: - - set -e - - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh - - eval $(sudo minikube docker-env) - - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - - docker tag sqlflow:ci sqlflow:submitter - - docker run --rm --net=host - -e SQLFLOW_WORKFLOW_STEP_IMAGE=sqlflow:submitter - -v /var/run/docker.sock:/var/run/docker.sock - -v $HOME/.kube:/root/.kube - -v /home/$USER/.minikube/:/home/$USER/.minikube/ - -v $TRAVIS_BUILD_DIR:/work -w /work - sqlflow:ci scripts/test/workflow.sh + # - stage: BuildAndTest + # script: + # - set -e + # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + # - docker run --rm -it -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:dev + # pre-commit run -a --show-diff-on-failure + # - env: SQLFLOW_TEST_DB=mysql + # script: + # - set -e + # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + # - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci + # scripts/test/units.sh + # - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci + # scripts/upload_coveralls.sh + # - env: SQLFLOW_TEST_DB=hive # run more parallel tests in the same stage: + # script: + # - set -e + # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + # - docker pull sqlflow/gohive:dev + # - docker run -d --name=hive sqlflow/gohive:dev python3 -m http.server 8899 + # - docker run --rm -it --net=container:hive + # -v $TRAVIS_BUILD_DIR:/work -w /work + # -v $GOPATH:/root/go + # sqlflow:ci scripts/test/hive.sh + # - env: SQLFLOW_TEST_DB=maxcompute + # script: + # - set -e + # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + # - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh + # - docker run --rm --net=host + # -v /var/run/docker.sock:/var/run/docker.sock + # -v $HOME/.kube:/root/.kube + # -v /home/$USER/.minikube/:/home/$USER/.minikube/ + # -v $TRAVIS_BUILD_DIR:/work -w /work + # sqlflow:ci scripts/test/ipython.sh + # - docker run --rm --net=host + # -e SQLFLOW_TEST_DB_MAXCOMPUTE_AK=$MAXCOMPUTE_AK + # -e SQLFLOW_TEST_DB_MAXCOMPUTE_SK=$MAXCOMPUTE_SK + # -v /var/run/docker.sock:/var/run/docker.sock + # -v $HOME/.kube:/root/.kube + # -v /home/$USER/.minikube/:/home/$USER/.minikube/ + # -v $TRAVIS_BUILD_DIR:/work -w /work + # sqlflow:ci scripts/test/maxcompute.sh + # - env: SQLFLOW_TEST=java + # script: + # - set -e + # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + # - docker run --rm + # -v $TRAVIS_BUILD_DIR:/work -w /work + # sqlflow:ci scripts/test/java.sh + # - env: SQLFLOW_TEST=workflow + # script: + # - set -e + # - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh + # - eval $(sudo minikube docker-env) + # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + # - docker tag sqlflow:ci sqlflow:submitter + # - docker run --rm --net=host + # -e SQLFLOW_WORKFLOW_STEP_IMAGE=sqlflow:submitter + # -v /var/run/docker.sock:/var/run/docker.sock + # -v $HOME/.kube:/root/.kube + # -v /home/$USER/.minikube/:/home/$USER/.minikube/ + # -v $TRAVIS_BUILD_DIR:/work -w /work + # sqlflow:ci scripts/test/workflow.sh - stage: Deploy - env: DESC="Deploy server Docker image" - script: - - $TRAVIS_BUILD_DIR/scripts/travis/deploy_server.sh - - env: DESC="Deploy macOS client" + env: DESC="Deploy macOS client" os: osx script: - $TRAVIS_BUILD_DIR/scripts/travis/deploy_client.sh @@ -114,3 +111,6 @@ jobs: os: windows script: - $TRAVIS_BUILD_DIR/scripts/travis/deploy_client.sh + # - env: DESC="Deploy server Docker image" + # script: + # - $TRAVIS_BUILD_DIR/scripts/travis/deploy_server.sh diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index 71bbf4377f..cf0e8b1eb3 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -x # DEBUG set -e # For more informaiton about deployment with Travis CI, please refer @@ -42,11 +43,11 @@ echo "Verify Go is installed ..." go env -echo "Install axel ..." +echo "Install download tools ..." case "$TRAVIS_OS_NAME" in linux) sudo apt-get -qq update > /dev/null - sudo apt-get -qq install -y axel > /dev/null + sudo apt-get -qq install -y axel unzip > /dev/null ;; windows) choco install axel ;; osx) brew install axel ;; @@ -73,11 +74,11 @@ echo "Install goyacc and protoc-gen-go ..." go get \ github.com/golang/protobuf/protoc-gen-go@v1.3.3 \ golang.org/x/tools/cmd/goyacc -sudo cp $GOPATH/bin/* /usr/local/bin/ +export PATH=$GOPATH/bin:$PATH echo "Build cmd/sqlflow into /tmp ..." -cd $TRAVIS_BUILD_DIR +cd "$TRAVIS_BUILD_DIR" go generate ./... GOBIN=/tmp go install ./cmd/sqlflow @@ -90,11 +91,11 @@ case "$TRAVIS_OS_NAME" in esac axel --quiet http://devtools.qiniu.com/$F.zip unzip $F.zip -sudo mv $F /usr/local/bin/qshell +export PATH=$PWD:$PATH echo "Publish /tmp/sqlflow to Qiniu Object Storage ..." qshell account "$QINIU_AK" "$QINIU_SK" "wu" qshell rput sqlflow-release \ - $RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow \ + "$RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow" \ /tmp/sqlflow From 2269342386765a904907d420b8eb596e06dfdfbe Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 12:06:32 -0700 Subject: [PATCH 02/16] Enable deploy from pull request for debug --- scripts/travis/deploy_client.sh | 41 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index cf0e8b1eb3..e18057fd33 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -19,25 +19,30 @@ set -e # For more informaiton about deployment with Travis CI, please refer # to the file header of deploy_docker.sh -if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then - echo "Skip deployment on pull request" - exit 0 -fi - -# Figure out the tag to push sqlflow:ci. -if [[ "$TRAVIS_BRANCH" == "develop" ]]; then - if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then - RELEASE_TAG="nightly" - else - RELEASE_TAG="latest" - fi -elif [[ "$TRAVIS_TAG" != "" ]]; then - RELEASE_TAG="$TRAVIS_TAG" -else - echo "Cannot figure out Docker image tag." - exit 1 -fi +# DEBUG +# +# if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then +# echo "Skip deployment on pull request" +# exit 0 +# fi +# DEBUG +# +# # Figure out the tag to push sqlflow:ci. +# if [[ "$TRAVIS_BRANCH" == "develop" ]]; then +# if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then +# RELEASE_TAG="nightly" +# else +# RELEASE_TAG="latest" +# fi +# elif [[ "$TRAVIS_TAG" != "" ]]; then +# RELEASE_TAG="$TRAVIS_TAG" +# else +# echo "Cannot figure out Docker image tag." +# exit 1 +# fi + +RELEASE_TAG="latest" # DEBUG echo "Verify Go is installed ..." go env From 50652df896a8b1c4ced762c196f0612e444e4fbe Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 12:11:00 -0700 Subject: [PATCH 03/16] make axel print --- scripts/travis/deploy_client.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index e18057fd33..ee827ad456 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -44,9 +44,6 @@ set -e RELEASE_TAG="latest" # DEBUG -echo "Verify Go is installed ..." -go env - echo "Install download tools ..." case "$TRAVIS_OS_NAME" in @@ -65,10 +62,8 @@ case "$TRAVIS_OS_NAME" in # The following code snippet comes from docker/dev/install.sh echo "Install protoc ..." PROTOC_SITE="https://github.com/protocolbuffers/protobuf/releases" - axel --quiet --output "p.zip" \ - $PROTOC_SITE"/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip" - sudo unzip -qq p.zip -d /usr/local - rm p.zip + axel $PROTOC_SITE"/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip" + sudo unzip -qq protoc-3.7.1-linux-x86_64.zip -d /usr/local ;; windows) choco install protoc ;; esac @@ -94,7 +89,7 @@ case "$TRAVIS_OS_NAME" in windows) F="qshell-windows-x64-v2.4.1.exe" ;; osx) F="qshell-darwin-x64-v2.4.1" ;; esac -axel --quiet http://devtools.qiniu.com/$F.zip +axel http://devtools.qiniu.com/$F.zip unzip $F.zip export PATH=$PWD:$PATH From 20529c36fb39a6516cb3d06df9b49844cd8b87e1 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 12:15:59 -0700 Subject: [PATCH 04/16] axel print version --- scripts/travis/deploy_client.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index ee827ad456..9bde817462 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -62,6 +62,7 @@ case "$TRAVIS_OS_NAME" in # The following code snippet comes from docker/dev/install.sh echo "Install protoc ..." PROTOC_SITE="https://github.com/protocolbuffers/protobuf/releases" + axel --version # debug axel $PROTOC_SITE"/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip" sudo unzip -qq protoc-3.7.1-linux-x86_64.zip -d /usr/local ;; From 2befc8af2819323ded038581e62c857dbb67515f Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 12:20:59 -0700 Subject: [PATCH 05/16] Use Ubuntu Bionic for deploy Linux --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 23b5e4c7bb..c902dff5bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,6 +105,7 @@ jobs: - $TRAVIS_BUILD_DIR/scripts/travis/deploy_client.sh - env: DESC="Deploy Linux client" os: linux + dist: bionic script: - $TRAVIS_BUILD_DIR/scripts/travis/deploy_client.sh - env: DESC="Deploy Windows client" From f5ee68875cd54cc8f0dcd251535908b6a3bb3913 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 12:31:34 -0700 Subject: [PATCH 06/16] use the correct qshell filename --- scripts/travis/deploy_client.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index 9bde817462..5f29106e74 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -92,6 +92,7 @@ case "$TRAVIS_OS_NAME" in esac axel http://devtools.qiniu.com/$F.zip unzip $F.zip +mv $F qshell # Use a shorter name export PATH=$PWD:$PATH From baf89e605743181e7e01793806bfadcefbb59f00 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:00:34 -0700 Subject: [PATCH 07/16] Try using 0 instead of syscall.Stdin with terminal.IsTerminal --- cmd/sqlflow/main.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/sqlflow/main.go b/cmd/sqlflow/main.go index b8f83d4131..d1a2ee234f 100644 --- a/cmd/sqlflow/main.go +++ b/cmd/sqlflow/main.go @@ -349,7 +349,13 @@ func main() { log.Fatalf("error SQLFLOW_DATASOURCE: %v", err) } - isTerminal := !flagPassed("execute", "e", "file", "f") && terminal.IsTerminal(syscall.Stdin) + // You might want to use syscall.Stdin instead of 0; however, + // unfortunately, we cannot. the syscall standard package has + // a special implementation for Windows, where the type of + // syscall.Stdin is not int as in Linux and macOS, but + // uintptr. + isTerminal := !flagPassed("execute", "e", "file", "f") && + terminal.IsTerminal(0) sqlFile := os.Stdin if flagPassed("file", "f") && *sqlFileName != "-" { From 7b89eab2b181295cea86c6da82bd7e8b0bd16424 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:04:47 -0700 Subject: [PATCH 08/16] Remove import syscall --- cmd/sqlflow/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/sqlflow/main.go b/cmd/sqlflow/main.go index d1a2ee234f..e6a43d9cdb 100644 --- a/cmd/sqlflow/main.go +++ b/cmd/sqlflow/main.go @@ -28,7 +28,6 @@ import ( "os" "path/filepath" "strings" - "syscall" "github.com/joho/godotenv" "golang.org/x/crypto/ssh/terminal" From bfee60f1ea7c76e977979d19207577489072e0da Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:10:42 -0700 Subject: [PATCH 09/16] GOBIN=$PWD otherwise Windows cnanot access the output --- scripts/travis/deploy_client.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index 5f29106e74..f5077c2afb 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -81,7 +81,7 @@ export PATH=$GOPATH/bin:$PATH echo "Build cmd/sqlflow into /tmp ..." cd "$TRAVIS_BUILD_DIR" go generate ./... -GOBIN=/tmp go install ./cmd/sqlflow +GOBIN=$PWD go install ./cmd/sqlflow echo "Install Qiniu client for $TRAVIS_OS_NAME ..." @@ -100,4 +100,4 @@ echo "Publish /tmp/sqlflow to Qiniu Object Storage ..." qshell account "$QINIU_AK" "$QINIU_SK" "wu" qshell rput sqlflow-release \ "$RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow" \ - /tmp/sqlflow + ./sqlflow From 8acd4b3467deaefa549374ce5ecac88baf4ec6a3 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:38:59 -0700 Subject: [PATCH 10/16] Use --overwrite with qshell rput --- scripts/travis/deploy_client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index f5077c2afb..559dd76fa4 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -98,6 +98,6 @@ export PATH=$PWD:$PATH echo "Publish /tmp/sqlflow to Qiniu Object Storage ..." qshell account "$QINIU_AK" "$QINIU_SK" "wu" -qshell rput sqlflow-release \ +qshell rput --overwrite sqlflow-release \ "$RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow" \ ./sqlflow From 3948265d166351458624f0ef36965d6ce01d77bc Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:40:47 -0700 Subject: [PATCH 11/16] Use $F instead of qshell because for windows it is qshell.exe --- scripts/travis/deploy_client.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index 559dd76fa4..bda3ae1424 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -92,12 +92,11 @@ case "$TRAVIS_OS_NAME" in esac axel http://devtools.qiniu.com/$F.zip unzip $F.zip -mv $F qshell # Use a shorter name export PATH=$PWD:$PATH echo "Publish /tmp/sqlflow to Qiniu Object Storage ..." -qshell account "$QINIU_AK" "$QINIU_SK" "wu" -qshell rput --overwrite sqlflow-release \ +$F account "$QINIU_AK" "$QINIU_SK" "wu" +$F rput --overwrite sqlflow-release \ "$RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow" \ ./sqlflow From 73b52753c8c80a224220e7a7aa004c7bb0a474a2 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:42:59 -0700 Subject: [PATCH 12/16] Shorter lines --- scripts/travis/deploy_client.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index bda3ae1424..b536b75b21 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -97,6 +97,7 @@ export PATH=$PWD:$PATH echo "Publish /tmp/sqlflow to Qiniu Object Storage ..." $F account "$QINIU_AK" "$QINIU_SK" "wu" -$F rput --overwrite sqlflow-release \ - "$RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow" \ - ./sqlflow +$F rput --overwrite \ + sqlflow-release \ + "$RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow" \ + ./sqlflow From fba6b9e37f35b2377d77eb78325c5fdf5ea659c6 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:47:21 -0700 Subject: [PATCH 13/16] brew install > /dev/null --- scripts/travis/deploy_client.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index b536b75b21..cc3f1d9732 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -52,7 +52,7 @@ case "$TRAVIS_OS_NAME" in sudo apt-get -qq install -y axel unzip > /dev/null ;; windows) choco install axel ;; - osx) brew install axel ;; + osx) brew install axel > /dev/null ;; esac @@ -62,7 +62,7 @@ case "$TRAVIS_OS_NAME" in # The following code snippet comes from docker/dev/install.sh echo "Install protoc ..." PROTOC_SITE="https://github.com/protocolbuffers/protobuf/releases" - axel --version # debug + axel --version # debug axel $PROTOC_SITE"/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip" sudo unzip -qq protoc-3.7.1-linux-x86_64.zip -d /usr/local ;; From bba9569b085d819a4f37c93aa332b6074d3cecf6 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 13:53:03 -0700 Subject: [PATCH 14/16] Upload sqlflow.exe for Windows --- scripts/travis/deploy_client.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index cc3f1d9732..dca07d3f67 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -80,8 +80,9 @@ export PATH=$GOPATH/bin:$PATH echo "Build cmd/sqlflow into /tmp ..." cd "$TRAVIS_BUILD_DIR" -go generate ./... -GOBIN=$PWD go install ./cmd/sqlflow +go generate ./... > /dev/null +mkdir "$PWD"/build +GOBIN="$PWD"/build go install ./cmd/sqlflow > /dev/null echo "Install Qiniu client for $TRAVIS_OS_NAME ..." @@ -100,4 +101,4 @@ $F account "$QINIU_AK" "$QINIU_SK" "wu" $F rput --overwrite \ sqlflow-release \ "$RELEASE_TAG/$TRAVIS_OS_NAME/sqlflow" \ - ./sqlflow + "$PWD"/build/sqlflow* # Need * because for Windows it is sqlflow.exe From 6194823292c90c0bca82d5116a3712e761c6988d Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 14:05:00 -0700 Subject: [PATCH 15/16] Silence deploy --- scripts/travis/deploy_client.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index dca07d3f67..560c66b924 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -x # DEBUG set -e # For more informaiton about deployment with Travis CI, please refer @@ -42,7 +41,7 @@ set -e # exit 1 # fi -RELEASE_TAG="latest" # DEBUG +RELEASE_TAG="latest" # debug echo "Install download tools ..." @@ -61,9 +60,8 @@ case "$TRAVIS_OS_NAME" in linux) # The following code snippet comes from docker/dev/install.sh echo "Install protoc ..." - PROTOC_SITE="https://github.com/protocolbuffers/protobuf/releases" - axel --version # debug - axel $PROTOC_SITE"/download/v3.7.1/protoc-3.7.1-linux-x86_64.zip" + PROTOC_SITE="https://github.com/protocolbuffers/protobuf/releases/" + axel --quiet $PROTOC_SITE"download/v3.7.1/protoc-3.7.1-linux-x86_64.zip" sudo unzip -qq protoc-3.7.1-linux-x86_64.zip -d /usr/local ;; windows) choco install protoc ;; @@ -74,7 +72,8 @@ protoc --version echo "Install goyacc and protoc-gen-go ..." go get \ github.com/golang/protobuf/protoc-gen-go@v1.3.3 \ - golang.org/x/tools/cmd/goyacc + golang.org/x/tools/cmd/goyacc \ + > /dev/null export PATH=$GOPATH/bin:$PATH @@ -91,7 +90,7 @@ case "$TRAVIS_OS_NAME" in windows) F="qshell-windows-x64-v2.4.1.exe" ;; osx) F="qshell-darwin-x64-v2.4.1" ;; esac -axel http://devtools.qiniu.com/$F.zip +axel --quiet http://devtools.qiniu.com/$F.zip unzip $F.zip export PATH=$PWD:$PATH From 3a6e1322f207ae0178f471c72d7ba97a68fd3eec Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 15 May 2020 14:15:51 -0700 Subject: [PATCH 16/16] move deploy to the second CI stage --- .travis.yml | 134 ++++++++++++++++---------------- scripts/travis/deploy_client.sh | 43 +++++----- 2 files changed, 86 insertions(+), 91 deletions(-) diff --git a/.travis.yml b/.travis.yml index c902dff5bb..eab3ce2da3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,70 +34,70 @@ install: - echo "skip install" jobs: include: - # - stage: BuildAndTest - # script: - # - set -e - # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - # - docker run --rm -it -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:dev - # pre-commit run -a --show-diff-on-failure - # - env: SQLFLOW_TEST_DB=mysql - # script: - # - set -e - # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - # - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci - # scripts/test/units.sh - # - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci - # scripts/upload_coveralls.sh - # - env: SQLFLOW_TEST_DB=hive # run more parallel tests in the same stage: - # script: - # - set -e - # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - # - docker pull sqlflow/gohive:dev - # - docker run -d --name=hive sqlflow/gohive:dev python3 -m http.server 8899 - # - docker run --rm -it --net=container:hive - # -v $TRAVIS_BUILD_DIR:/work -w /work - # -v $GOPATH:/root/go - # sqlflow:ci scripts/test/hive.sh - # - env: SQLFLOW_TEST_DB=maxcompute - # script: - # - set -e - # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - # - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh - # - docker run --rm --net=host - # -v /var/run/docker.sock:/var/run/docker.sock - # -v $HOME/.kube:/root/.kube - # -v /home/$USER/.minikube/:/home/$USER/.minikube/ - # -v $TRAVIS_BUILD_DIR:/work -w /work - # sqlflow:ci scripts/test/ipython.sh - # - docker run --rm --net=host - # -e SQLFLOW_TEST_DB_MAXCOMPUTE_AK=$MAXCOMPUTE_AK - # -e SQLFLOW_TEST_DB_MAXCOMPUTE_SK=$MAXCOMPUTE_SK - # -v /var/run/docker.sock:/var/run/docker.sock - # -v $HOME/.kube:/root/.kube - # -v /home/$USER/.minikube/:/home/$USER/.minikube/ - # -v $TRAVIS_BUILD_DIR:/work -w /work - # sqlflow:ci scripts/test/maxcompute.sh - # - env: SQLFLOW_TEST=java - # script: - # - set -e - # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - # - docker run --rm - # -v $TRAVIS_BUILD_DIR:/work -w /work - # sqlflow:ci scripts/test/java.sh - # - env: SQLFLOW_TEST=workflow - # script: - # - set -e - # - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh - # - eval $(sudo minikube docker-env) - # - $TRAVIS_BUILD_DIR/scripts/travis/build.sh - # - docker tag sqlflow:ci sqlflow:submitter - # - docker run --rm --net=host - # -e SQLFLOW_WORKFLOW_STEP_IMAGE=sqlflow:submitter - # -v /var/run/docker.sock:/var/run/docker.sock - # -v $HOME/.kube:/root/.kube - # -v /home/$USER/.minikube/:/home/$USER/.minikube/ - # -v $TRAVIS_BUILD_DIR:/work -w /work - # sqlflow:ci scripts/test/workflow.sh + - stage: BuildAndTest + script: + - set -e + - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + - docker run --rm -it -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:dev + pre-commit run -a --show-diff-on-failure + - env: SQLFLOW_TEST_DB=mysql + script: + - set -e + - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci + scripts/test/units.sh + - docker run --rm -v $TRAVIS_BUILD_DIR:/work -w /work sqlflow:ci + scripts/upload_coveralls.sh + - env: SQLFLOW_TEST_DB=hive # run more parallel tests in the same stage: + script: + - set -e + - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + - docker pull sqlflow/gohive:dev + - docker run -d --name=hive sqlflow/gohive:dev python3 -m http.server 8899 + - docker run --rm -it --net=container:hive + -v $TRAVIS_BUILD_DIR:/work -w /work + -v $GOPATH:/root/go + sqlflow:ci scripts/test/hive.sh + - env: SQLFLOW_TEST_DB=maxcompute + script: + - set -e + - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh + - docker run --rm --net=host + -v /var/run/docker.sock:/var/run/docker.sock + -v $HOME/.kube:/root/.kube + -v /home/$USER/.minikube/:/home/$USER/.minikube/ + -v $TRAVIS_BUILD_DIR:/work -w /work + sqlflow:ci scripts/test/ipython.sh + - docker run --rm --net=host + -e SQLFLOW_TEST_DB_MAXCOMPUTE_AK=$MAXCOMPUTE_AK + -e SQLFLOW_TEST_DB_MAXCOMPUTE_SK=$MAXCOMPUTE_SK + -v /var/run/docker.sock:/var/run/docker.sock + -v $HOME/.kube:/root/.kube + -v /home/$USER/.minikube/:/home/$USER/.minikube/ + -v $TRAVIS_BUILD_DIR:/work -w /work + sqlflow:ci scripts/test/maxcompute.sh + - env: SQLFLOW_TEST=java + script: + - set -e + - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + - docker run --rm + -v $TRAVIS_BUILD_DIR:/work -w /work + sqlflow:ci scripts/test/java.sh + - env: SQLFLOW_TEST=workflow + script: + - set -e + - $TRAVIS_BUILD_DIR/scripts/travis/setup_minikube.sh + - eval $(sudo minikube docker-env) + - $TRAVIS_BUILD_DIR/scripts/travis/build.sh + - docker tag sqlflow:ci sqlflow:submitter + - docker run --rm --net=host + -e SQLFLOW_WORKFLOW_STEP_IMAGE=sqlflow:submitter + -v /var/run/docker.sock:/var/run/docker.sock + -v $HOME/.kube:/root/.kube + -v /home/$USER/.minikube/:/home/$USER/.minikube/ + -v $TRAVIS_BUILD_DIR:/work -w /work + sqlflow:ci scripts/test/workflow.sh - stage: Deploy env: DESC="Deploy macOS client" os: osx @@ -112,6 +112,6 @@ jobs: os: windows script: - $TRAVIS_BUILD_DIR/scripts/travis/deploy_client.sh - # - env: DESC="Deploy server Docker image" - # script: - # - $TRAVIS_BUILD_DIR/scripts/travis/deploy_server.sh + - env: DESC="Deploy server Docker image" + script: + - $TRAVIS_BUILD_DIR/scripts/travis/deploy_server.sh diff --git a/scripts/travis/deploy_client.sh b/scripts/travis/deploy_client.sh index 560c66b924..7ab87a704b 100755 --- a/scripts/travis/deploy_client.sh +++ b/scripts/travis/deploy_client.sh @@ -18,30 +18,25 @@ set -e # For more informaiton about deployment with Travis CI, please refer # to the file header of deploy_docker.sh -# DEBUG -# -# if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then -# echo "Skip deployment on pull request" -# exit 0 -# fi - -# DEBUG -# -# # Figure out the tag to push sqlflow:ci. -# if [[ "$TRAVIS_BRANCH" == "develop" ]]; then -# if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then -# RELEASE_TAG="nightly" -# else -# RELEASE_TAG="latest" -# fi -# elif [[ "$TRAVIS_TAG" != "" ]]; then -# RELEASE_TAG="$TRAVIS_TAG" -# else -# echo "Cannot figure out Docker image tag." -# exit 1 -# fi - -RELEASE_TAG="latest" # debug +if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then + echo "Skip deployment on pull request" + exit 0 +fi + + +# Figure out the tag to push sqlflow:ci. +if [[ "$TRAVIS_BRANCH" == "develop" ]]; then + if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then + RELEASE_TAG="nightly" + else + RELEASE_TAG="latest" + fi +elif [[ "$TRAVIS_TAG" != "" ]]; then + RELEASE_TAG="$TRAVIS_TAG" +else + echo "Cannot figure out Docker image tag." + exit 1 +fi echo "Install download tools ..."