From 8d0774ccf89372cbcb8bc1991fc58d0c4e3a306d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=84=E6=9C=A8?= Date: Tue, 10 Sep 2019 09:48:38 +0800 Subject: [PATCH 1/3] debug develop CI fail --- sql/codegen.go | 16 ++++++++-------- sql/executor.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/codegen.go b/sql/codegen.go index 2bfb0eec95..86b0e8452c 100644 --- a/sql/codegen.go +++ b/sql/codegen.go @@ -117,6 +117,13 @@ func newFiller(pr *extendedSelect, ds *trainAndValDataset, fts fieldTypes, db *D if err != nil { return nil, err } + if err == nil && r.Driver == "hive" { + // remove the last ';' which leads to a (hive)ParseException + r.TrainingDatasetSQL = strings.TrimSuffix(r.TrainingDatasetSQL, ";") + r.ValidationDatasetSQL = strings.TrimSuffix(r.ValidationDatasetSQL, ";") + r.PredictionDatasetSQL = strings.TrimSuffix(r.PredictionDatasetSQL, ";") + } + trainResolved, err := resolveTrainClause(&pr.trainClause, &pr.standardSelect, r.connectionConfig) if err != nil { return nil, err @@ -229,14 +236,7 @@ func newFiller(pr *extendedSelect, ds *trainAndValDataset, fts fieldTypes, db *D return nil, e } } - - if err == nil && r.Driver == "hive" { - // remove the last ';' which leads to a (hive)ParseException - r.TrainingDatasetSQL = strings.TrimSuffix(r.TrainingDatasetSQL, ";") - r.ValidationDatasetSQL = strings.TrimSuffix(r.ValidationDatasetSQL, ";") - r.PredictionDatasetSQL = strings.TrimSuffix(r.PredictionDatasetSQL, ";") - } - return r, err + return r, e } func newConnectionConfig(db *DB) (*connectionConfig, error) { diff --git a/sql/executor.go b/sql/executor.go index b1dc56b50f..a9b0fb4f4d 100644 --- a/sql/executor.go +++ b/sql/executor.go @@ -353,7 +353,7 @@ func (cw *logChanWriter) Write(p []byte) (n int, err error) { if err := cw.wr.Write(cw.prev); err != nil { return len(cw.prev), err } - + log.Debugf("Train script output: %s", cw.prev) cw.prev = "" } return n, nil From 88ff4b1a353cd2c3c835b8a4031fbd7f86ce0861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=84=E6=9C=A8?= Date: Tue, 10 Sep 2019 10:52:37 +0800 Subject: [PATCH 2/3] fix develop build --- scripts/image_build.sh | 4 +++- sql/executor.go | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/image_build.sh b/scripts/image_build.sh index 7df5b0d4fd..30f96f46a3 100644 --- a/scripts/image_build.sh +++ b/scripts/image_build.sh @@ -25,6 +25,7 @@ echo ". /miniconda/etc/profile.d/conda.sh" >> ~/.bashrc echo "source activate sqlflow-dev" >> ~/.bashrc # keras.datasets.imdb only works with numpy==1.16.1 +# NOTE: shap == 0.30.1 depends on dill but not include dill as it's dependency, need to install manually source /miniconda/bin/activate sqlflow-dev && python -m pip install \ numpy==1.16.1 \ tensorflow==${TENSORFLOW_VERSION} \ @@ -36,6 +37,7 @@ notebook==6.0.0 \ sqlflow==0.4.0 \ pre-commit \ odps \ +dill \ shap \ ${PIP_ADD_PACKAGES} @@ -121,7 +123,7 @@ pip install xgboost==0.90 # 10. install Hadoop to use as the client when writing CSV to hive tables HADOOP_URL=https://archive.apache.org/dist/hadoop/common/stable/hadoop-${HADOOP_VERSION}.tar.gz -curl -fsSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz +curl -fsSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz tar -xzf /tmp/hadoop.tar.gz -C /opt/ rm -rf /tmp/hadoop.tar.gz rm -rf /opt/hadoop-${HADOOP_VERSION}/share/doc diff --git a/sql/executor.go b/sql/executor.go index a9b0fb4f4d..fe1ffa52bd 100644 --- a/sql/executor.go +++ b/sql/executor.go @@ -353,7 +353,6 @@ func (cw *logChanWriter) Write(p []byte) (n int, err error) { if err := cw.wr.Write(cw.prev); err != nil { return len(cw.prev), err } - log.Debugf("Train script output: %s", cw.prev) cw.prev = "" } return n, nil From da43d7f331522a1f9ed5cadecc820ae61be2cac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=84=E6=9C=A8?= Date: Tue, 10 Sep 2019 11:15:49 +0800 Subject: [PATCH 3/3] update --- sql/codegen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/codegen.go b/sql/codegen.go index 86b0e8452c..bdbb7e4228 100644 --- a/sql/codegen.go +++ b/sql/codegen.go @@ -117,7 +117,7 @@ func newFiller(pr *extendedSelect, ds *trainAndValDataset, fts fieldTypes, db *D if err != nil { return nil, err } - if err == nil && r.Driver == "hive" { + if r.Driver == "hive" { // remove the last ';' which leads to a (hive)ParseException r.TrainingDatasetSQL = strings.TrimSuffix(r.TrainingDatasetSQL, ";") r.ValidationDatasetSQL = strings.TrimSuffix(r.ValidationDatasetSQL, ";")