From da096a9b2fff4d4ad43040741931f45ddcf1c0de Mon Sep 17 00:00:00 2001 From: Kirill Morozov Date: Fri, 9 Sep 2016 11:43:27 +0100 Subject: [PATCH 1/4] BIGTOP-2530. Create Greenplum packages --- .../src/common/gpdb/do-component-build | 21 +++++++ .../src/common/gpdb/do-component-configure | 19 ++++++ .../src/common/gpdb/install_gpdb.sh | 19 ++++++ bigtop-packages/src/deb/gpdb/changelog | 1 + bigtop-packages/src/deb/gpdb/compat | 1 + bigtop-packages/src/deb/gpdb/control | 24 +++++++ bigtop-packages/src/deb/gpdb/copyright | 15 +++++ bigtop-packages/src/deb/gpdb/rules | 44 +++++++++++++ bigtop-packages/src/deb/gpdb/source/format | 1 + bigtop-packages/src/rpm/gpdb/BUILD/.gitignore | 0 bigtop-packages/src/rpm/gpdb/RPMS/.gitignore | 0 .../src/rpm/gpdb/SOURCES/.gitignore | 0 bigtop-packages/src/rpm/gpdb/SPECS/gpdb.spec | 63 +++++++++++++++++++ bigtop-packages/src/rpm/gpdb/SRPMS/.gitignore | 0 bigtop.bom | 10 +++ bigtop_toolchain/manifests/packages.pp | 19 +++++- build.gradle | 3 +- pom.xml | 1 + 18 files changed, 237 insertions(+), 4 deletions(-) create mode 100644 bigtop-packages/src/common/gpdb/do-component-build create mode 100644 bigtop-packages/src/common/gpdb/do-component-configure create mode 100755 bigtop-packages/src/common/gpdb/install_gpdb.sh create mode 100644 bigtop-packages/src/deb/gpdb/changelog create mode 100644 bigtop-packages/src/deb/gpdb/compat create mode 100644 bigtop-packages/src/deb/gpdb/control create mode 100644 bigtop-packages/src/deb/gpdb/copyright create mode 100755 bigtop-packages/src/deb/gpdb/rules create mode 100644 bigtop-packages/src/deb/gpdb/source/format create mode 100644 bigtop-packages/src/rpm/gpdb/BUILD/.gitignore create mode 100644 bigtop-packages/src/rpm/gpdb/RPMS/.gitignore create mode 100644 bigtop-packages/src/rpm/gpdb/SOURCES/.gitignore create mode 100644 bigtop-packages/src/rpm/gpdb/SPECS/gpdb.spec create mode 100644 bigtop-packages/src/rpm/gpdb/SRPMS/.gitignore diff --git a/bigtop-packages/src/common/gpdb/do-component-build b/bigtop-packages/src/common/gpdb/do-component-build new file mode 100644 index 0000000000..2ecb755cb4 --- /dev/null +++ b/bigtop-packages/src/common/gpdb/do-component-build @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +rm -f gpdb + +make diff --git a/bigtop-packages/src/common/gpdb/do-component-configure b/bigtop-packages/src/common/gpdb/do-component-configure new file mode 100644 index 0000000000..6f567b1b97 --- /dev/null +++ b/bigtop-packages/src/common/gpdb/do-component-configure @@ -0,0 +1,19 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +./configure --prefix=$1 diff --git a/bigtop-packages/src/common/gpdb/install_gpdb.sh b/bigtop-packages/src/common/gpdb/install_gpdb.sh new file mode 100755 index 0000000000..d3c7986c13 --- /dev/null +++ b/bigtop-packages/src/common/gpdb/install_gpdb.sh @@ -0,0 +1,19 @@ +#!/bin/bash -x +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +make install DESTDIR=$1 diff --git a/bigtop-packages/src/deb/gpdb/changelog b/bigtop-packages/src/deb/gpdb/changelog new file mode 100644 index 0000000000..d4858cd496 --- /dev/null +++ b/bigtop-packages/src/deb/gpdb/changelog @@ -0,0 +1 @@ +--- This is auto-generated diff --git a/bigtop-packages/src/deb/gpdb/compat b/bigtop-packages/src/deb/gpdb/compat new file mode 100644 index 0000000000..7f8f011eb7 --- /dev/null +++ b/bigtop-packages/src/deb/gpdb/compat @@ -0,0 +1 @@ +7 diff --git a/bigtop-packages/src/deb/gpdb/control b/bigtop-packages/src/deb/gpdb/control new file mode 100644 index 0000000000..b0bb744117 --- /dev/null +++ b/bigtop-packages/src/deb/gpdb/control @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Source: gpdb +Section: misc +Priority: extra +Maintainer: Bigtop +Standards-Version: 4.3.99.0 +Homepage: https://github.com/greenplum-db/gpdb + +Package: gpdb +Architecture: all +Description: Greenplum MPP database engine diff --git a/bigtop-packages/src/deb/gpdb/copyright b/bigtop-packages/src/deb/gpdb/copyright new file mode 100644 index 0000000000..eca1af7dba --- /dev/null +++ b/bigtop-packages/src/deb/gpdb/copyright @@ -0,0 +1,15 @@ +Format: http://dep.debian.net/deps/dep5 +Source: https://github.com/greenplum-db/gpdb +Upstream-Name: Greenplum + +Files: * +Copyright: 2010-2011, The Apache Software Foundation +License: Apache-2.0 + +Files debian/* +Copyright: 2011, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/bigtop-packages/src/deb/gpdb/rules b/bigtop-packages/src/deb/gpdb/rules new file mode 100755 index 0000000000..7caf747e19 --- /dev/null +++ b/bigtop-packages/src/deb/gpdb/rules @@ -0,0 +1,44 @@ +#!/usr/bin/make -f +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +%: + dh $@ + +override_dh_auto_clean: + +override_dh_auto_configure: + bash debian/do-component-configure /usr/lib/gpdb + +override_dh_auto_build: + bash debian/do-component-build + +override_dh_auto_install: + bash -x debian/install_gpdb.sh /tmp + install -d -p -m 755 debian/gpdb/usr/lib/gpdb + cp -r --parents /tmp/usr/lib/gpdb debian/gpdb/usr/lib/gpdb + rm -rf /tmp/usr/lib/gpdb + +override_dh_auto_test: + @echo Tests are disabled diff --git a/bigtop-packages/src/deb/gpdb/source/format b/bigtop-packages/src/deb/gpdb/source/format new file mode 100644 index 0000000000..163aaf8d82 --- /dev/null +++ b/bigtop-packages/src/deb/gpdb/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/bigtop-packages/src/rpm/gpdb/BUILD/.gitignore b/bigtop-packages/src/rpm/gpdb/BUILD/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bigtop-packages/src/rpm/gpdb/RPMS/.gitignore b/bigtop-packages/src/rpm/gpdb/RPMS/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bigtop-packages/src/rpm/gpdb/SOURCES/.gitignore b/bigtop-packages/src/rpm/gpdb/SOURCES/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bigtop-packages/src/rpm/gpdb/SPECS/gpdb.spec b/bigtop-packages/src/rpm/gpdb/SPECS/gpdb.spec new file mode 100644 index 0000000000..b2e082a4a3 --- /dev/null +++ b/bigtop-packages/src/rpm/gpdb/SPECS/gpdb.spec @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +%define man_dir %{_mandir} + +%if %{?suse_version:1}0 +%define bin_gpdb /usr/lib/gpdb +%define doc_gpdb %{_docdir}/%{name} +%define autorequire no +%else +%define bin_gpdb /usr/lib/gpdb +%define doc_gpdb %{_docdir}/%{name}-%{gpdb_version} +%define autorequire yes +%endif +%define debug_package %{nil} + +Name: gpdb +Version: %{gpdb_version} +Release: %{gpdb_release} +Summary: Greenplum MPP database enginer +URL: https://github.com/greenplum-db/gpdb +Group: Development/Libraries +Buildroot: %{_topdir}/INSTALL/%{name}-%{version} +License: ASL 2.0 +Source0: gpdb-%{gpdb_version}.tar.gz +Source1: do-component-build +Source2: install_gpdb.sh +Source3: do-component-configure +AutoReqProv: %{autorequire} + +%description +gpdb + +%prep +%autosetup + +%build +bash %{SOURCE3} %{bin_gpdb} +bash %{SOURCE1} + +%install +%__rm -rf $RPM_BUILD_ROOT +bash %{SOURCE2} %{_tmppath} +mkdir -p $RPM_BUILD_ROOT%{bin_gpdb} +cp -f -r %{_tmppath}%{bin_gpdb}/* $RPM_BUILD_ROOT/%{bin_gpdb} +%__rm -rf %{_tmppath}%{bin_gpdb} + +%files +%defattr(-,root,root) +%{bin_gpdb} + +%changelog diff --git a/bigtop-packages/src/rpm/gpdb/SRPMS/.gitignore b/bigtop-packages/src/rpm/gpdb/SRPMS/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bigtop.bom b/bigtop.bom index 1b0a96bf64..96eb4aba81 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -444,5 +444,15 @@ bigtop { url { site = "https://github.com/quantcast/qfs/archive" archive = site } } + 'gpdb' { + name = 'gpdb' + relNotes = 'GreenPlum' + version { base = '4.3.99.0'; pkg = base; release = 1 } + tarball { source = "master.zip" + destination = "$name-${version.base}.tar.gz" } + url { download_path = "pub/greenplum/$name/" + site = "https://github.com/greenplum-db/gpdb/archive" + archive = "site" } + } } } diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp index 319610d989..94fbbf37b2 100644 --- a/bigtop_toolchain/manifests/packages.pp +++ b/bigtop_toolchain/manifests/packages.pp @@ -22,15 +22,28 @@ } else { $mysql_devel="mariadb-devel" } - $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "bzip2-devel" ] + $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "bzip2-devel", "readline-devel", "ncurses-devel", "libidn-devel", "libcurl-devel", "libevent-devel", "apr-devel", "bison", "perl-Env" ] } - /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "libbz2-devel" ] + /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "libbz2-devel", "libcurl-devel", "bison", "flex"] # fix package dependencies: BIGTOP-2120 and BIGTOP-2152 and BIGTOP-2471 exec { '/usr/bin/zypper -n install --force-resolution krb5 libopenssl-devel': } -> Package <| |> + # fix package libapr1 + exec { 'suse_12.3_repo': + command => '/usr/bin/zypper ar --no-gpgcheck http://download.opensuse.org/distribution/12.3/repo/oss/suse/ libapr1', + unless => "/usr/bin/zypper lr | grep -q libapr1", + } + package { 'libapr1': + ensure => '1.4.6', + require => [Exec['suse_12.3_repo']] + } + package { 'libapr1-devel': + ensure => '1.4.6', + require => [Package['libapr1']] + } } Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel", "bzip2-devel" ] } - /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev", "libbz2-dev" ] + /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev", "libbz2-dev", "libreadline6", "libreadline6-dev", "zlib1g", "libapr1", "libapr1-dev", "libevent-dev", "libcurl4-gnutls-dev", "bison", "flex", "python-dev" ] exec { "apt-update": command => "/usr/bin/apt-get update" diff --git a/build.gradle b/build.gradle index f0fe7d7223..9c6f46e84d 100644 --- a/build.gradle +++ b/build.gradle @@ -129,7 +129,8 @@ rat { "bigtop-deploy/puppet/README.md", "bigtop-deploy/puppet/modules/qfs/README.md", "bigtop-deploy/puppet/config/site.csv.example", - "dl/**" + "dl/**", + "bigtop-deploy/puppet/modules/gpdb/templates/**" ] } diff --git a/pom.xml b/pom.xml index fb038f5324..b25bce773b 100644 --- a/pom.xml +++ b/pom.xml @@ -359,6 +359,7 @@ bigtop-deploy/puppet/modules/qfs/README.md bigtop-deploy/puppet/config/site.csv.example dl/** + bigtop-deploy/puppet/modules/gpdb/templates/** From 03eac0a612186cb6e8c6b03e4f56e0e7138868fa Mon Sep 17 00:00:00 2001 From: Kirill Morozov Date: Fri, 9 Sep 2016 11:47:31 +0100 Subject: [PATCH 2/4] BIGTOP-2531. Create Greenplum deployment scripts --- .../puppet/hieradata/bigtop/cluster.yaml | 6 + bigtop-deploy/puppet/hieradata/site.yaml | 1 + bigtop-deploy/puppet/manifests/cluster.pp | 7 +- bigtop-deploy/puppet/modules/gpdb/README.md | 19 + .../puppet/modules/gpdb/manifests/init.pp | 396 ++++++++++++++++++ .../puppet/modules/gpdb/templates/gp_dbid | 3 + .../puppet/modules/gpdb/templates/gpssh.conf | 12 + .../puppet/modules/gpdb/templates/init-db.sh | 4 + .../insert-to-faultStrategy-table.sh | 2 + .../insert-to-segmentConfig-table.sh | 2 + .../modules/gpdb/templates/postmaster.opts | 1 + .../start-master-db-in-admin-mode.sh | 2 + .../puppet/modules/gpdb/templates/stop-db.sh | 2 + .../modules/gpdb/templates/test-master-db.sh | 2 + .../puppet/modules/gpdb/tests/init.pp | 23 + provisioner/docker/docker-hadoop.sh | 2 +- 16 files changed, 482 insertions(+), 2 deletions(-) create mode 100644 bigtop-deploy/puppet/modules/gpdb/README.md create mode 100644 bigtop-deploy/puppet/modules/gpdb/manifests/init.pp create mode 100755 bigtop-deploy/puppet/modules/gpdb/templates/gp_dbid create mode 100755 bigtop-deploy/puppet/modules/gpdb/templates/gpssh.conf create mode 100755 bigtop-deploy/puppet/modules/gpdb/templates/init-db.sh create mode 100755 bigtop-deploy/puppet/modules/gpdb/templates/insert-to-faultStrategy-table.sh create mode 100755 bigtop-deploy/puppet/modules/gpdb/templates/insert-to-segmentConfig-table.sh create mode 100755 bigtop-deploy/puppet/modules/gpdb/templates/postmaster.opts create mode 100755 bigtop-deploy/puppet/modules/gpdb/templates/start-master-db-in-admin-mode.sh create mode 100644 bigtop-deploy/puppet/modules/gpdb/templates/stop-db.sh create mode 100644 bigtop-deploy/puppet/modules/gpdb/templates/test-master-db.sh create mode 100644 bigtop-deploy/puppet/modules/gpdb/tests/init.pp diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 60c59b1d53..c747b4e783 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -203,3 +203,9 @@ flink::common::jobmanager_port: "6123" flink::common::ui_port: "8081" flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}" + +gpdb::common::nodes: ["master_node_fqdn", "segment_node_fqdn"] +gpdb::common::gp_home: "/usr/lib/gpdb" +gpdb::common::db_base_dir: "/data_gp" +gpdb::common::master_db_port: "5432" +gpdb::common::segment_db_port_prefix: "4000" diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index b9ed6f98e2..7ee434d798 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -37,6 +37,7 @@ hadoop::hadoop_storage_dirs: # - yarn # - zookeeper # - ycsb +# - gpdb #bigtop::roles_enabled: false diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 12e0424829..7144f02ba5 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -117,6 +117,10 @@ worker => ["qfs-chunkserver"], client => ["qfs-client"], }, + gpdb => { + master => ["gpdb-master"], + worker => ["gpdb-segment"], + }, kafka => { master => ["kafka-server"], } @@ -189,7 +193,8 @@ "ycsb", "kerberos", "zeppelin", - "kafka" + "kafka", + "gpdb" ] deploy_module { $modules: diff --git a/bigtop-deploy/puppet/modules/gpdb/README.md b/bigtop-deploy/puppet/modules/gpdb/README.md new file mode 100644 index 0000000000..bc0cfcf3ea --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/README.md @@ -0,0 +1,19 @@ +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +# Package instalation notes + +Greenplum scripts need the following python libs: psutil, paramiko. +And they need python-pip package, which in its turn needs libffi-dev, python-lockfile. diff --git a/bigtop-deploy/puppet/modules/gpdb/manifests/init.pp b/bigtop-deploy/puppet/modules/gpdb/manifests/init.pp new file mode 100644 index 0000000000..98475a380c --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/manifests/init.pp @@ -0,0 +1,396 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class gpdb { + + class deploy($roles) { + if ("gpdb-master" in $roles or "gpdb-segment" in $roles) { + include gpdb::common + } + } + + class common($nodes, $gp_home, $db_base_dir, $master_db_port, $segment_db_port_prefix) { + + include stdlib + + class { 'gpdb::common::install_packages': } + + class { 'gpdb::common::gpadmin_user':} + + class { 'gpdb::common::prepare_file_structure': + base_dir => "$db_base_dir", + require => Class['gpdb::common::gpadmin_user'] + } + + class { 'gpdb::common::prepare_db_dirs': + base_dir => "$db_base_dir", + nodes => $nodes, + gp_home => $gp_home, + master_port => $master_db_port, + segment_port_prefix => $segment_db_port_prefix, + require => [Class['gpdb::common::prepare_file_structure'], Package["gpdb"]] + } + + gpdb::server{"stop_if_running": + nodes => $gpdb::common::nodes, + gp_home => $gpdb::common::gp_home, + db_base_dir => $gpdb::common::db_base_dir, + master_db_port => $gpdb::common::master_db_port, + segment_db_port_prefix => $gpdb::common::segment_db_port_prefix, + require => Class['gpdb::common::prepare_db_dirs'], + start_or_stop => stopped, + } + + class { 'gpdb::common::configure_master_node': + base_dir => "$db_base_dir", + nodes => $nodes, + gp_home => $gp_home, + master_port => $master_db_port, + segment_port_prefix => $segment_db_port_prefix, + require => Class['gpdb::common::prepare_db_dirs'] + } + + class { 'gpdb::common::stop_master_in_admin_mode': + base_dir => "$db_base_dir", + gp_home => $gp_home, + master_port => $master_db_port, + require => Class['gpdb::common::configure_master_node'] + } + + gpdb::server{"start": + nodes => $gpdb::common::nodes, + gp_home => $gpdb::common::gp_home, + db_base_dir => $gpdb::common::db_base_dir, + master_db_port => $gpdb::common::master_db_port, + segment_db_port_prefix => $gpdb::common::segment_db_port_prefix, + require => Class['gpdb::common::stop_master_in_admin_mode'], + start_or_stop => running, + } + + class gpadmin_user{ + group { 'gpadmin': + ensure => 'present', + } + user { 'gpadmin': + ensure => 'present', + system => false, + managehome => true, + groups => 'gpadmin', + shell => '/bin/bash', + } + } + + class install_packages{ + case $operatingsystem{ + /(?i:(centos|fedora))/: { + yumrepo { "epel": + baseurl => "http://download.fedoraproject.org/pub/epel/7/\$basearch", + descr => "epel packages", + enabled => 1, + gpgcheck => 0, + } + package { ["libffi-devel"]: + ensure => latest, + } + package { ["python-lockfile"]: + ensure => latest, + } + package { ["psutil"]: + ensure => latest, + provider => pip, + require => Package["python-pip"], + } + package { ["paramiko"]: + ensure => latest, + provider => pip, + require => Package["python-pip"], + } + package { ["python-pip"]: + ensure => latest, + require => [ + Yumrepo["epel"], + Package["libffi-devel"], + Package["python-lockfile"], + ], + } + } + /(?i:(SLES|opensuse))/: { + } + Amazon: { } + /(Ubuntu|Debian)/: { + package { ["libffi-dev"]: + ensure => latest, + } + package { ["python-lockfile"]: + ensure => latest, + } + package { ["psutil"]: + ensure => latest, + provider => pip, + require => Package["python-pip"], + } + package { ["paramiko"]: + ensure => latest, + provider => pip, + require => Package["python-pip"], + } + package { ["python-pip"]: + ensure => latest, + require => [ + Package["libffi-dev"], + Package["python-lockfile"], + ], + } + } + } + package { ["gpdb"]: + ensure => latest, + } + } + + class prepare_file_structure($base_dir = undef){ + file { "$base_dir": + ensure => 'directory', + owner => 'gpadmin', + group => 'gpadmin', + } + file { "$base_dir/master": + ensure => 'directory', + owner => 'gpadmin', + group => 'gpadmin', + } + file { "$base_dir/primary": + ensure => 'directory', + owner => 'gpadmin', + group => 'gpadmin', + } + } + + class prepare_db_dirs($nodes = undef, $base_dir = undef, $gp_home = undef, $master_port = undef, $segment_port_prefix = undef){ + file { '/home/gpadmin/init-db.sh': + content => template('gpdb/init-db.sh'), + require => [ + File["$base_dir/primary"], + File["$base_dir/master"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + file { '/home/gpadmin/stop-db.sh': + content => template('gpdb/stop-db.sh'), + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + each($nodes) |$index, $value| { + notice($value) + if ($::fqdn == $value) { + $dbid = $index+1 + $content = $index-1 + $hostname = $value + $segment_nodes_count=size($nodes)-1 + if ($dbid == 1){ + $port = $master_port + $db_sub_dir = "master" + $m_options = "master" + $x_options = "-x 0 -E" + $z_options = "$segment_nodes_count" + } else{ + $port = "$segment_port_prefix${content}" + $db_sub_dir = "primary" + $m_options = "mirrorless" + $x_options = "" + $z_options = "$segment_nodes_count" + } + $db_dir = "$base_dir/${db_sub_dir}/gpseg${content}" + exec { "create_master_db${db_dir}": + command => "init-db.sh $db_dir", + require => [File["/home/gpadmin/init-db.sh"]], + path => '/home/gpadmin', + user => 'gpadmin', + } + file_line { "allow all host connection${db_dir}": + path => "${db_dir}/pg_hba.conf", + line => 'host all all 0.0.0.0/0 trust', + require => [Exec["create_master_db${db_dir}"]], + } + file_line { "add 1 conf${db_dir}": + path => "${db_dir}/postgresql.conf", + line => 'log_statement=all', + require => [Exec["create_master_db${db_dir}"]], + } + file_line { "add 2 conf${db_dir}": + path => "${db_dir}/postgresql.conf", + line => 'checkpoint_segments=8', + require => [Exec["create_master_db${db_dir}"]], + } + file_line { "add 3 conf${db_dir}": + path => "${db_dir}/postgresql.conf", + line => "port=${port}", + require => [Exec["create_master_db${db_dir}"]], + } + file { "${db_dir}/gp_dbid": + content => template('gpdb/gp_dbid'), + require => [Exec["create_master_db${db_dir}"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + file { "${db_dir}/gpssh.conf": + content => template('gpdb/gpssh.conf'), + require => [Exec["create_master_db${db_dir}"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + file { "${db_dir}/postmaster.opts": + content => template('gpdb/postmaster.opts'), + require => [Exec["create_master_db${db_dir}"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + file { "${db_dir}/run.opts": + source => "${db_dir}/postmaster.opts", + ensure => present, + require => [File["${db_dir}/postmaster.opts"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + } + } + } + + class stop_master_in_admin_mode($base_dir = undef, $gp_home = undef, $master_port = undef){ + exec { 'stop-master-db-in-admin-mode': + command => "stop-db.sh $base_dir/master/gpseg-1 $master_port", + path => '/home/gpadmin', + user => 'gpadmin', + require => [ + Exec["create_master_db$base_dir/master/gpseg-1"], + File['/home/gpadmin/stop-db.sh'] + ], + } + } + + class configure_master_node($nodes = undef, $base_dir = undef, $gp_home = undef, $master_port = undef, $segment_port_prefix = undef){ + if ($::fqdn == $nodes[0]) { + notice("must make admin") + file { '/home/gpadmin/start-master-db-in-admin-mode.sh': + content => template('gpdb/start-master-db-in-admin-mode.sh'), + require => [Exec["create_master_db$base_dir/master/gpseg-1"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + file { '/home/gpadmin/test-master-db.sh': + content => template('gpdb/test-master-db.sh'), + require => [Exec["create_master_db$base_dir/master/gpseg-1"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + exec { 'start-master-db-in-admin-mode': + command => "start-master-db-in-admin-mode.sh $base_dir/master/gpseg-1", + path => '/home/gpadmin', + user => 'gpadmin', + require => [ + Exec["create_master_db$base_dir/master/gpseg-1"], + File['/home/gpadmin/start-master-db-in-admin-mode.sh'] + ], + } + file { '/home/gpadmin/insert-to-segmentConfig-table.sh': + content => template('gpdb/insert-to-segmentConfig-table.sh'), + require => [Exec["start-master-db-in-admin-mode"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + file { '/home/gpadmin/insert-to-faultStrategy-table.sh': + content => template('gpdb/insert-to-faultStrategy-table.sh'), + require => [Exec["start-master-db-in-admin-mode"]], + owner => 'gpadmin', + group => 'gpadmin', + mode => '0700', + } + exec { "insert-to-faultStrategy-table": + environment => ["PGOPTIONS=-c gp_session_role=utility"], + command => "insert-to-faultStrategy-table.sh", + path => '/home/gpadmin', + user => 'gpadmin', + require => [ + File["/home/gpadmin/insert-to-faultStrategy-table.sh"], + Exec['start-master-db-in-admin-mode'], + ], + } + each($nodes) |$index, $value| { + $dbid = $index+1 + $content = $index-1 + $hostname = $value + if ($dbid == 1){ + $port = $master_port + $db_sub_dir = "master" + $m_options = "master" + $x_options = "\"-x\" \"0\" \"-c\" \"gp_role=utility\"" + $z_options = "0" + } else{ + $port = "$segment_port_prefix${content}" + $db_sub_dir = "primary" + $m_options = "mirrorless" + $x_options = "" + $z_options = "3" + } + $db_dir = "$base_dir/${db_sub_dir}/gpseg${content}" + notice("${dbid} - ${content} - ${port} - ${hostname} - ${db_dir}") + exec { "insert-to-segmentConfig-table${dbid}": + environment => ["PGOPTIONS=-c gp_session_role=utility"], + command => "insert-to-segmentConfig-table.sh ${dbid} ${content} ${port} ${value} ${db_dir}", + path => '/home/gpadmin', + user => 'gpadmin', + require => [ + File["/home/gpadmin/insert-to-segmentConfig-table.sh"], + Exec['start-master-db-in-admin-mode'], + ], + } + } + } + } + } + + define server($nodes, $gp_home, $db_base_dir, $master_db_port, $segment_db_port_prefix, $start_or_stop){ + each($nodes) |$index, $value| { + $content = $index-1 + if ($::fqdn == $value) { + if ($index == 0){ + $db_sub_dir = "master" + $port = $master_db_port + }else{ + $db_sub_dir = "primary" + $port = "$segment_db_port_prefix${content}" + } + $db_dir = "$db_base_dir/${db_sub_dir}/gpseg${content}" + service { "gpdb${content}${start_or_stop}": + provider => base, + ensure => $start_or_stop, + start => "su -l gpadmin -m $db_dir/run.opts", + stop => "su -l gpadmin -m /home/gpadmin/stop-db.sh $db_dir $port", + hasstatus => false, + pattern => "${db_dir}", + } + } + } + } +} diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/gp_dbid b/bigtop-deploy/puppet/modules/gpdb/templates/gp_dbid new file mode 100755 index 0000000000..2066b20458 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/gp_dbid @@ -0,0 +1,3 @@ +# Greenplum Database identifier for this master/segment. +# Do not change the contents of this file. +dbid = <%= dbid %> diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/gpssh.conf b/bigtop-deploy/puppet/modules/gpdb/templates/gpssh.conf new file mode 100755 index 0000000000..640747f5ee --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/gpssh.conf @@ -0,0 +1,12 @@ +[gpssh] +# delaybeforesend specifies the time in seconds to wait at the +# beginning of an ssh interaction before doing anything. +# Increasing this value can have a big runtime impact at the +# beginning of gpssh. +delaybeforesend = 0.05 + +# prompt_validation_timeout specifies a timeout multiplier that +# will be used in validating the ssh prompt. Increasing this +# value will have a small runtime impact at the beginning of +# gpssh. +prompt_validation_timeout = 1.0 diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/init-db.sh b/bigtop-deploy/puppet/modules/gpdb/templates/init-db.sh new file mode 100755 index 0000000000..b1945267b3 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/init-db.sh @@ -0,0 +1,4 @@ +#!/bin/sh +if [ ! -d $1 ]; then + export LD_LIBRARY_PATH=<%= gp_home%>/lib:/lib;<%= gp_home%>/bin/initdb -E UNICODE -D $1 --locale=en_US.utf8 --max_connections=750 --shared_buffers=128000kB --is_filerep_mirrored=no --backend_output=$1.initdb +fi diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/insert-to-faultStrategy-table.sh b/bigtop-deploy/puppet/modules/gpdb/templates/insert-to-faultStrategy-table.sh new file mode 100755 index 0000000000..aeff932961 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/insert-to-faultStrategy-table.sh @@ -0,0 +1,2 @@ +#!/bin/sh +<%= gp_home%>/bin/psql -p <%= master_port%> -d "template1" -c "insert into gp_fault_strategy(fault_strategy) values ('n');" || true diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/insert-to-segmentConfig-table.sh b/bigtop-deploy/puppet/modules/gpdb/templates/insert-to-segmentConfig-table.sh new file mode 100755 index 0000000000..9bde305933 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/insert-to-segmentConfig-table.sh @@ -0,0 +1,2 @@ +#!/bin/sh +<%= gp_home%>/bin/psql -p <%= master_port%> -d "template1" -c "INSERT INTO gp_segment_configuration (dbid, content, role, preferred_role, mode, status, hostname, address, port, replication_port) VALUES ($1, $2, 'p', 'p', 's', 'u', '$4', '$4', $3, null);insert into pg_filespace_entry (fsefsoid, fsedbid, fselocation) values (3052, $1, '$5');" || true \ No newline at end of file diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/postmaster.opts b/bigtop-deploy/puppet/modules/gpdb/templates/postmaster.opts new file mode 100755 index 0000000000..96e2212af1 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/postmaster.opts @@ -0,0 +1 @@ +<%= gp_home%>/bin/postgres "-D" "<%= db_dir %>" "-p" "<%= port %>" "-b" "<%= dbid %>" "-z" "<%= z_options %>" "--silent-mode=true" "-i" "-M" "<%= m_options %>" "-C" "<%= content %>" <%= x_options %> diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/start-master-db-in-admin-mode.sh b/bigtop-deploy/puppet/modules/gpdb/templates/start-master-db-in-admin-mode.sh new file mode 100755 index 0000000000..3966e417f8 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/start-master-db-in-admin-mode.sh @@ -0,0 +1,2 @@ +#!/bin/bash +export PGPORT=<%= master_port%>;<%= gp_home%>/bin/pg_ctl -w -l $1/pg_log/startup.log -D $1 -o "-i -p <%= master_port%> -c gp_role=utility -M master -b 1 -C -1 -z 0 -m" start \ No newline at end of file diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/stop-db.sh b/bigtop-deploy/puppet/modules/gpdb/templates/stop-db.sh new file mode 100644 index 0000000000..5394b8a292 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/stop-db.sh @@ -0,0 +1,2 @@ +#!/bin/sh +<%= gp_home%>/bin/pg_ctl -p $2 -D $1 stop \ No newline at end of file diff --git a/bigtop-deploy/puppet/modules/gpdb/templates/test-master-db.sh b/bigtop-deploy/puppet/modules/gpdb/templates/test-master-db.sh new file mode 100644 index 0000000000..141cd3da60 --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/templates/test-master-db.sh @@ -0,0 +1,2 @@ +#!/bin/sh +<%= gp_home%>/bin/psql -p <%= master_port%> -d "template1" -c "select count(*) from gp_segment_configuration;" \ No newline at end of file diff --git a/bigtop-deploy/puppet/modules/gpdb/tests/init.pp b/bigtop-deploy/puppet/modules/gpdb/tests/init.pp new file mode 100644 index 0000000000..4a328a373d --- /dev/null +++ b/bigtop-deploy/puppet/modules/gpdb/tests/init.pp @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +gpdb::server{"start": + nodes => ["master_node_fqdn", "segment_node_fqdn"], + gp_home => "/usr/lib/gpdb", + db_base_dir => "/data_gp", + master_db_port => "5432", + segment_db_port_prefix => "4000", + start_or_stop => running, +} diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index 5e8e09dfc6..9a74162ebc 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -131,7 +131,7 @@ destroy() { } bigtop-puppet() { - docker exec $1 bash -c 'puppet apply --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp' + docker exec $1 bash -c 'puppet apply --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests' } get-yaml-config() { From 8435af76cf0c454940b58fa68478be083b275b3d Mon Sep 17 00:00:00 2001 From: Kirill Morozov Date: Fri, 9 Sep 2016 11:48:32 +0100 Subject: [PATCH 3/4] BIGTOP-2532. Create Greenplum test --- bigtop-tests/smoke-tests/gpdb/TestGpdb.groovy | 57 +++++++++++++++++++ bigtop-tests/smoke-tests/gpdb/build.gradle | 32 +++++++++++ 2 files changed, 89 insertions(+) create mode 100644 bigtop-tests/smoke-tests/gpdb/TestGpdb.groovy create mode 100644 bigtop-tests/smoke-tests/gpdb/build.gradle diff --git a/bigtop-tests/smoke-tests/gpdb/TestGpdb.groovy b/bigtop-tests/smoke-tests/gpdb/TestGpdb.groovy new file mode 100644 index 0000000000..0fb03cedea --- /dev/null +++ b/bigtop-tests/smoke-tests/gpdb/TestGpdb.groovy @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.junit.BeforeClass +import org.junit.AfterClass + +import org.apache.bigtop.itest.shell.Shell +import static org.junit.Assert.assertFalse +import org.junit.Test +import org.apache.commons.logging.LogFactory +import org.apache.commons.logging.Log + +import static org.apache.bigtop.itest.LogErrorsUtils.logError + +class TestGpdb { + static private Log LOG = LogFactory.getLog(Object.class) + + static Shell sh = new Shell("/bin/bash -s") + + @BeforeClass + static void setUp() { + // noop for now. + } + + @AfterClass + public static void tearDown() { + // noop for now + } + + @Test + void testGpdb() { + // Basic test to verify that the server is running, and is in a + // state that we expect. + LOG.info('Test gp_segment_configuration'); + sh.exec("runuser -l gpadmin /home/gpadmin/test-master-db.sh"); + logError(sh); + int count = sh.getOut()[2].trim().toInteger() + assertFalse(count == 0); + LOG.info('Test gp_segment_configuration finished'); + LOG.info('Configured segments count is '+count); + } +} diff --git a/bigtop-tests/smoke-tests/gpdb/build.gradle b/bigtop-tests/smoke-tests/gpdb/build.gradle new file mode 100644 index 0000000000..d0a9f96de6 --- /dev/null +++ b/bigtop-tests/smoke-tests/gpdb/build.gradle @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +def tests_to_include() { + return ["TestGpdb.groovy"]; +} + +sourceSets { + test { + groovy { + srcDirs = ["./"] + exclude { FileTreeElement elem -> (doExclude(elem.getName())) } + } + } +} + +test.doFirst { +} From 6610adc67ee98563961a574468242316cfc8fbac Mon Sep 17 00:00:00 2001 From: Kirill Morozov Date: Thu, 13 Oct 2016 07:48:23 +0100 Subject: [PATCH 4/4] cluster run script --- clusterRun.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 clusterRun.sh diff --git a/clusterRun.sh b/clusterRun.sh new file mode 100755 index 0000000000..b37cff8088 --- /dev/null +++ b/clusterRun.sh @@ -0,0 +1,3 @@ +#!/bin/sh +rsync -a --delete bigtop-deploy/puppet/hieradata/site.yaml bigtop-deploy/puppet/hieradata/bigtop /etc/puppet/hieradata/ +sudo puppet apply -d --parser future --modulepath="bigtop-deploy/puppet/modules:/etc/puppet/modules" bigtop-deploy/puppet/manifests