From 6d4bb998a264dc07b5d69403cfce0c64c4029129 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 21 Jul 2020 19:08:59 +0800 Subject: [PATCH 1/4] SUBMARINE-574. Add a script to initialize database --- docs/database/README.md | 26 +++++++++------- docs/database/init-database.sh | 52 ++++++++++++++++++++++++++++++++ docs/database/submarine-data.sql | 14 ++++++++- 3 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 docs/database/init-database.sh diff --git a/docs/database/README.md b/docs/database/README.md index c6ef8f7eb5..816e8172f5 100644 --- a/docs/database/README.md +++ b/docs/database/README.md @@ -32,9 +32,13 @@ Must: By using the official docker image of submarine database, only one docker command is required to run submarine database ```bash -docker run -it -p 3306:3306 -d --name submarine-database -e MYSQL_ROOT_PASSWORD=password apache/submarine:database-0.3.0 +docker run -it -p 3306:3306 -d --name submarine-database -e MYSQL_ROOT_PASSWORD=password apache/submarine:database-0.4.0 +``` +## Initialize submarine database +It will create users and tables that submarine requires +```shell script +sudo ./init-database ``` - ## Manual operation of the submarine database ### Modify character set (Optional) @@ -45,10 +49,10 @@ If you need to store Chinese character data in mysql, you need to execute the fo ``` bash > mysql -uroot -ppassword - + mysql>SHOW VARIABLES LIKE 'character_set_%'; // View database character set mysql>SHOW VARIABLES LIKE 'collation_%'; - + SET NAMES 'utf8'; ``` @@ -58,21 +62,21 @@ If you need to store Chinese character data in mysql, you need to execute the fo # install vim apt-get update apt-get install vim - + vi /etc/mysql/mysql.conf.d/mysqld.cnf - + [mysqld] character_set_server = utf8 - + [mysql] default-character-set = utf8 - + [mysql.server] default-character-set = utf8 - + [mysqld_safe] default-character-set = utf8 - + [client] default-character-set = utf8 ``` @@ -142,7 +146,7 @@ mysqldump -umetastore -ppassword metastore > metastore.sql; ## Travis -1. In the submarine's Travis, the `test database`, `database name`, `username` and `password` will be automatically created based on the contents of this document. +1. In the submarine's Travis, the `test database`, `database name`, `username` and `password` will be automatically created based on the contents of this document. Therefore, do not modify the database's `database name`, `username` and `password` configuration to avoid introducing some problems. diff --git a/docs/database/init-database.sh b/docs/database/init-database.sh new file mode 100644 index 0000000000..b8d5df4105 --- /dev/null +++ b/docs/database/init-database.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FWDIR="$(cd "$(dirname "$0")"; pwd)" +cd "$FWDIR" + +mysql -e CREATE USER 'submarine'@'%' IDENTIFIED BY 'password'; +mysql -e GRANT ALL PRIVILEGES ON * . * TO 'submarine'@'%'; + +mysql -e CREATE USER 'metastore'@'%' IDENTIFIED BY 'password'; +mysql -e GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%'; + +mysql -e CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test'; +mysql -e GRANT ALL PRIVILEGES ON * . * TO 'submarine_test'@'%'; + +mysql -e CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test'; +mysql -e GRANT ALL PRIVILEGES ON * . * TO 'metastore_test'@'%'; + +mysql -e "create database submarine_test;" +mysql -e "CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';" +mysql -e "use submarine_test; source ./submarine.sql; show tables;" + +mysql -e "create database metastore_test;" +mysql -e "CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore_test'@'%';" +mysql -e "use metastore_test; source ./metastore.sql; show tables;" + +mysql -e "create database submarine;" +mysql -e "CREATE USER 'submarine'@'%' IDENTIFIED BY 'password';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';" +mysql -e "use submarine; source ./submarine.sql; source ./submarine-data.sql; show tables;" + +mysql -e "create database metastore;" +mysql -e "CREATE USER 'metastore'@'%' IDENTIFIED BY 'password';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore'@'%';" +mysql -e "use metastore; source ./metastore.sql; show tables;" diff --git a/docs/database/submarine-data.sql b/docs/database/submarine-data.sql index 2c230abce0..f6d07d66b8 100644 --- a/docs/database/submarine-data.sql +++ b/docs/database/submarine-data.sql @@ -55,7 +55,6 @@ INSERT INTO `sys_department` VALUES ('1bc0cd98c8d311e98edc0242ac110002','AAA','G -- ---------------------------- INSERT INTO `sys_user` VALUES ('e9ca23d68d884d4ebb19d07889727dae', 'admin', 'administrator', '21232f297a57a5a743894a0e4a801fc3', 'avatar.png', '2018-12-05 00:00:00', NULL, 'dev@submarine.org', '18566666661', NULL, NULL, NULL, 1, 'admin', '2019-07-05 14:47:22', 'admin', '2019-07-05 14:47:22'); - -- ---------------------------- -- Records of team -- ---------------------------- @@ -78,3 +77,16 @@ INSERT INTO `params` (`id`, `key`, `value`, `worker_index`, `job_name`) VALUES (14, 'max_iter', '100', 'worker-1', 'application_123651651'), (15, 'n_jobs', '5', 'worker-1', 'application_123456898'), (16, 'alpha', '20', 'worker-1', 'application_123456789'); + +-- ---------------------------- +-- Records of environment +-- ---------------------------- +INSERT INTO `environment` VALUES ('environment_1595134205164_0002', 'my-submarine-env', + '{"name":"my-submarine-env", + "dockerImage":"continuumio/anaconda3", + "kernelSpec":{"name":"team_default_python_3.7", + "channels":["defaults"], + "dependencies":["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0","anaconda=2020.02=py37_0", + "anaconda-client=1.7.2=py37_0","anaconda-navigator=1.9.12=py37_0"]}}', + 'admin', '2020-05-06 14:00:05', 'Jack', '2020-05-06 14:00:14'); From a9fdaf757e1a96da55da79b0d22367367403aafe Mon Sep 17 00:00:00 2001 From: pingsutw Date: Tue, 21 Jul 2020 19:31:21 +0800 Subject: [PATCH 2/4] Update script --- .travis.yml | 18 +----------------- docs/database/README.md | 16 ++++++++-------- docs/database/init-database.sh | 31 ++++++++----------------------- docs/database/submarine-data.sql | 10 +--------- 4 files changed, 18 insertions(+), 57 deletions(-) mode change 100644 => 100755 docs/database/init-database.sh diff --git a/.travis.yml b/.travis.yml index 65ebf6fac4..eeae5a90aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,23 +63,7 @@ env: before_install: # mysql - sudo service mysql restart - - mysql -e "create database submarine_test;" - - mysql -e "CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';" - - mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';" - - mysql -e "use submarine_test; source ./docs/database/submarine.sql; show tables;" - - mysql -e "create database metastore_test;" - - mysql -e "CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test';" - - mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore_test'@'%';" - - mysql -e "use metastore_test; source ./docs/database/metastore.sql; show tables;" - # For e2e tests - - mysql -e "create database submarine;" - - mysql -e "CREATE USER 'submarine'@'%' IDENTIFIED BY 'password';" - - mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';" - - mysql -e "use submarine; source ./docs/database/submarine.sql; source ./docs/database/submarine-data.sql; show tables;" - - mysql -e "create database metastore;" - - mysql -e "CREATE USER 'metastore'@'%' IDENTIFIED BY 'password';" - - mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore'@'%';" - - mysql -e "use metastore; source ./docs/database/metastore.sql; show tables;" + - ./docs/database/init-database.sh - ./dev-support/travis/install_external_dependencies.sh # protobuf 3.10.1 - PROTOBUF_VERSION=3.10.1 diff --git a/docs/database/README.md b/docs/database/README.md index 816e8172f5..3fddaae417 100644 --- a/docs/database/README.md +++ b/docs/database/README.md @@ -97,15 +97,15 @@ Development database for development environment. ``` # in mysql container bash > mysql -uroot -ppassword -mysql> CREATE USER 'submarine'@'%' IDENTIFIED BY 'password'; +mysql> CREATE USER IF NOT EXISTS 'submarine'@'%' IDENTIFIED BY 'password'; mysql> GRANT ALL PRIVILEGES ON * . * TO 'submarine'@'%'; -mysql> CREATE DATABASE submarine CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> CREATE DATABASE IF NOT EXISTS submarine CHARACTER SET utf8 COLLATE utf8_general_ci; mysql> use submarine; mysql> source /submarine.sql; mysql> source /submarine-data.sql; -mysql> CREATE USER 'metastore'@'%' IDENTIFIED BY 'password'; +mysql> CREATE USER IF NOT EXISTS 'metastore'@'%' IDENTIFIED BY 'password'; mysql> GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%'; -mysql> CREATE DATABASE metastore CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> CREATE DATABASE IF NOT EXISTS metastore CHARACTER SET utf8 COLLATE utf8_general_ci; mysql> use metastore; mysql> source /metastore.sql; mysql> quit @@ -121,14 +121,14 @@ Test database for program unit testing and Travis test environment. ``` # in mysql container bash > mysql -uroot -ppassword -mysql> CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test'; +mysql> CREATE USER IF NOT EXISTS 'submarine_test'@'%' IDENTIFIED BY 'password_test'; mysql> GRANT ALL PRIVILEGES ON * . * TO 'submarine_test'@'%'; -mysql> CREATE DATABASE `submarine_test` CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> CREATE DATABASE IF NOT EXISTS `submarine_test` CHARACTER SET utf8 COLLATE utf8_general_ci; mysql> use `submarine_test`; mysql> source /submarine.sql; -mysql> CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test'; +mysql> CREATE USER IF NOT EXISTS 'metastore_test'@'%' IDENTIFIED BY 'password_test'; mysql> GRANT ALL PRIVILEGES ON * . * TO 'metastore_test'@'%'; -mysql> CREATE DATABASE `metastore_test` CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> CREATE DATABASE IF NOT EXISTS `metastore_test` CHARACTER SET utf8 COLLATE utf8_general_ci; mysql> use `metastore_test`; mysql> source /metastore.sql; mysql> quit diff --git a/docs/database/init-database.sh b/docs/database/init-database.sh old mode 100644 new mode 100755 index b8d5df4105..0ef64c0d16 --- a/docs/database/init-database.sh +++ b/docs/database/init-database.sh @@ -16,37 +16,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -FWDIR="$(cd "$(dirname "$0")"; pwd)" -cd "$FWDIR" - -mysql -e CREATE USER 'submarine'@'%' IDENTIFIED BY 'password'; -mysql -e GRANT ALL PRIVILEGES ON * . * TO 'submarine'@'%'; - -mysql -e CREATE USER 'metastore'@'%' IDENTIFIED BY 'password'; -mysql -e GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%'; - -mysql -e CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test'; -mysql -e GRANT ALL PRIVILEGES ON * . * TO 'submarine_test'@'%'; - -mysql -e CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test'; -mysql -e GRANT ALL PRIVILEGES ON * . * TO 'metastore_test'@'%'; - -mysql -e "create database submarine_test;" -mysql -e "CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "CREATE DATABASE IF NOT EXISTS submarine_test;" +mysql -e "CREATE USER IF NOT EXISTS 'submarine_test'@'%' IDENTIFIED BY 'password_test';" mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';" mysql -e "use submarine_test; source ./submarine.sql; show tables;" -mysql -e "create database metastore_test;" -mysql -e "CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "CREATE DATABASE IF NOT EXISTS metastore_test;" +mysql -e "CREATE USER IF NOT EXISTS 'metastore_test'@'%' IDENTIFIED BY 'password_test';" mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore_test'@'%';" mysql -e "use metastore_test; source ./metastore.sql; show tables;" -mysql -e "create database submarine;" -mysql -e "CREATE USER 'submarine'@'%' IDENTIFIED BY 'password';" +mysql -e "CREATE DATABASE IF NOT EXISTS submarine;" +mysql -e "CREATE USER IF NOT EXISTS 'submarine'@'%' IDENTIFIED BY 'password';" mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';" mysql -e "use submarine; source ./submarine.sql; source ./submarine-data.sql; show tables;" -mysql -e "create database metastore;" -mysql -e "CREATE USER 'metastore'@'%' IDENTIFIED BY 'password';" +mysql -e "CREATE DATABASE IF NOT EXISTS metastore;" +mysql -e "CREATE USER IF NOT EXISTS 'metastore'@'%' IDENTIFIED BY 'password';" mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore'@'%';" mysql -e "use metastore; source ./metastore.sql; show tables;" diff --git a/docs/database/submarine-data.sql b/docs/database/submarine-data.sql index f6d07d66b8..c0e2a6d4d2 100644 --- a/docs/database/submarine-data.sql +++ b/docs/database/submarine-data.sql @@ -81,12 +81,4 @@ INSERT INTO `params` (`id`, `key`, `value`, `worker_index`, `job_name`) VALUES -- ---------------------------- -- Records of environment -- ---------------------------- -INSERT INTO `environment` VALUES ('environment_1595134205164_0002', 'my-submarine-env', - '{"name":"my-submarine-env", - "dockerImage":"continuumio/anaconda3", - "kernelSpec":{"name":"team_default_python_3.7", - "channels":["defaults"], - "dependencies":["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", - "alabaster=0.7.12=py37_0","anaconda=2020.02=py37_0", - "anaconda-client=1.7.2=py37_0","anaconda-navigator=1.9.12=py37_0"]}}', - 'admin', '2020-05-06 14:00:05', 'Jack', '2020-05-06 14:00:14'); +INSERT INTO `environment` VALUES ('environment_1595134205164_0002', 'my-submarine-env','{"name":"my-submarine-env","dockerImage":"continuumio/anaconda3","kernelSpec":{"name":"team_default_python_3.7","channels":["defaults"],"dependencies":["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0","alabaster=0.7.12=py37_0","anaconda=2020.02=py37_0","anaconda-client=1.7.2=py37_0","anaconda-navigator=1.9.12=py37_0"]}}','admin', '2020-05-06 14:00:05', 'Jack', '2020-05-06 14:00:14'); From a91b5cfb130a88278a30e6b48146d727b6c25085 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 22 Jul 2020 13:45:36 +0800 Subject: [PATCH 3/4] Fix error --- docs/database/init-database.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/database/init-database.sh b/docs/database/init-database.sh index 0ef64c0d16..60bf15f623 100755 --- a/docs/database/init-database.sh +++ b/docs/database/init-database.sh @@ -16,6 +16,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +FWDIR="$(cd "$(dirname "$0")"; pwd)" +cd "$FWDIR" + mysql -e "CREATE DATABASE IF NOT EXISTS submarine_test;" mysql -e "CREATE USER IF NOT EXISTS 'submarine_test'@'%' IDENTIFIED BY 'password_test';" mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';" From d49d7fcd70feb6e6c281cf2e77765f677ab7cefa Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 22 Jul 2020 15:08:47 +0800 Subject: [PATCH 4/4] Update environment sql --- docs/database/submarine-data.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/database/submarine-data.sql b/docs/database/submarine-data.sql index c0e2a6d4d2..2e93c6e124 100644 --- a/docs/database/submarine-data.sql +++ b/docs/database/submarine-data.sql @@ -81,4 +81,4 @@ INSERT INTO `params` (`id`, `key`, `value`, `worker_index`, `job_name`) VALUES -- ---------------------------- -- Records of environment -- ---------------------------- -INSERT INTO `environment` VALUES ('environment_1595134205164_0002', 'my-submarine-env','{"name":"my-submarine-env","dockerImage":"continuumio/anaconda3","kernelSpec":{"name":"team_default_python_3.7","channels":["defaults"],"dependencies":["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0","alabaster=0.7.12=py37_0","anaconda=2020.02=py37_0","anaconda-client=1.7.2=py37_0","anaconda-navigator=1.9.12=py37_0"]}}','admin', '2020-05-06 14:00:05', 'Jack', '2020-05-06 14:00:14'); +INSERT INTO `environment` VALUES ('environment_1595134205164_0002', 'my-submarine-test-env','{"name":"my-submarine-env","dockerImage":"continuumio/anaconda3","kernelSpec":{"name":"team_default_python_3.7","channels":["defaults"],"dependencies":["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0","alabaster=0.7.12=py37_0","anaconda=2020.02=py37_0","anaconda-client=1.7.2=py37_0","anaconda-navigator=1.9.12=py37_0"]}}','admin', '2020-05-06 14:00:05', 'Jack', '2020-05-06 14:00:14');