From cede655dbecb141356a1c2753ca3d51bd6e8205b Mon Sep 17 00:00:00 2001 From: Vishnu Gajendran Date: Mon, 13 Apr 2015 20:45:37 +0000 Subject: [PATCH] BIGTOP-1792. Add hive-server2 and hive-metastore services to bigtop hive deployment --- .../puppet/hieradata/bigtop/cluster.yaml | 34 +++- bigtop-deploy/puppet/manifests/cluster.pp | 9 +- .../modules/hadoop-hive/manifests/init.pp | 145 +++++++++++++++++- .../hadoop-hive/templates/hive-site.xml | 54 +++++-- 4 files changed, 224 insertions(+), 18 deletions(-) diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 41d477bc1..94a2a947a 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -10,7 +10,7 @@ # FQDN of standby node (enables HA if set) #bigtop::hadoop_standby_head_node: "standbyNN.example.com" # FQDN of gateway node (if separate from NN) -#bigtop::hadoop_gateway_node: "gateway.example.com" +bigtop::hadoop_gateway_node: "%{hiera('bigtop::hadoop_head_node')}" # A list of stack' components to be deployed can be specified via special # "$components" list. If $components isn't set then everything in the stack will @@ -157,7 +157,37 @@ hue::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" giraph::client::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}" -hadoop-hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_config::zookeeper_quorum')}" +hadoop-hive::common::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_config::zookeeper_quorum')}" + +hadoop-hive::common::metastore_server_host: "%{hiera('bigtop::hadoop_gateway_node')}" +hadoop-hive::common::metastore_server_port: "9083" +hadoop-hive::common::metastore_server_uris: + - "thrift://%{hiera('hadoop-hive::common::metastore_server_host')}:%{hiera('hadoop-hive::common::metastore_server_port')}" + +hadoop-hive::common::hdfs_uri: "%{hiera('bigtop::hadoop_namenode_uri')}" +hadoop-hive::common::metastore_database_host: "%{hiera('bigtop-mysql::server::host')}" +hadoop-hive::common::metastore_database_port: "%{hiera('bigtop-mysql::server::port')}" + +# Bigtop supports multiple databases as metastore database. +# Possible values for metastore database type: derby, mysql (default) +hadoop-hive::common::metastore_database_type: "mysql" +hadoop-hive::common::metastore_database_name: "hive" +hadoop-hive::common::metastore_database_user: "hive" +hadoop-hive::common::metastore_database_password: "hive" + +bigtop-mysql::server::host: "%{hiera('bigtop::hadoop_gateway_node')}" +bigtop-mysql::server::port: "3306" +bigtop-mysql::server::users: + "%{hiera('hadoop-hive::common::metastore_database_user')}@%": + password_hash: "*4DF1D66463C18D44E3B001A8FB1BBFBEA13E27FC" +bigtop-mysql::server::grants: + "%{hiera('hadoop-hive::common::metastore_database_user')}@%/*.*": + options: + - "GRANT" + privileges: + - "ALL" + table: "*.*" + user: "%{hiera('hadoop-hive::common::metastore_database_user')}@%" # tez hadoop::common::tez_conf_dir: "/etc/tez/conf" diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 15a247b5e..ea2e6c852 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -91,6 +91,7 @@ }, hive => { client => ["hive-client"], + gateway_server => ["hive-server", "hive-metastore-server"], }, tez => { client => ["tez-client"], @@ -102,6 +103,10 @@ ycsb => { client => ["ycsb-client"], }, + bigtop-mysql => { + master => ["mysql-server"], + client => ["mysql-client"], + }, } class hadoop_cluster_node ( @@ -140,13 +145,15 @@ } class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_node { + define deploy_module($roles) { class { "${name}::deploy": - roles => $roles, + roles => $roles, } } $modules = [ + "bigtop-mysql", "crunch", "giraph", "hadoop", diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp index b0eee0a91..5cf62bb64 100644 --- a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp @@ -16,22 +16,155 @@ class hadoop-hive { class deploy ($roles) { - if ("hive-client" in $roles) { + + if ('hive-client' in $roles) { include hadoop-hive::client } + if ('hive-metastore-server' in $roles) { + include hadoop-hive::metastore-server + } + if ('hive-server' in $roles) { + include hadoop-hive::server + if ('hive-metastore-server' in $roles) { + Class['Hadoop-hive::Metastore-server'] -> Class['Hadoop-hive::Server'] + } + } + + # Need to make sure local mysql server is setup correctly (in case hive is + # using it) before initializing the schema + if ('hive-client' or 'hive-metastore-server' or 'hive-server' in $roles) { + if ('mysql-server' in $roles) { + Class['Bigtop-mysql::Server'] -> Exec<| title == 'init hive-metastore schema' |> + } + } } - class client($hbase_master = "", - $hbase_zookeeper_quorum = "", - $hive_execution_engine = "mr") { + class common ( + $metastore_server_uris = [], + $metastore_database_type = 'derby', + $metastore_database_host = $fqdn, + $metastore_database_port = '3306', + $metastore_database_name = 'hive', + $metastore_database_user = 'hive', + $metastore_database_password = 'hive', + $hbase_master = undef, + $hbase_zookeeper_quorum = undef, + $hdfs_uri = undef, + $hive_site_overrides = {}, + $hive_execution_engine = "mr" + ) { + + $metastore_database_configs = { + derby => { + url => "jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true", + driver_class => 'org.apache.derby.jdbc.EmbeddedDriver', + }, + mysql => { + url => "jdbc:mysql://${metastore_database_host}:${metastore_database_port}/${metastore_database_name}?createDatabaseIfNotExist=true", + driver_class => 'com.mysql.jdbc.Driver', + } + } + $supported_database_types = keys($metastore_database_configs) + + if (! ($metastore_database_type in $supported_database_types)) { + fail("hadoop-hive::metastore_database_type: $database_type is not supported. Supported database types are ", $supported_database_types) + } - package { "hive": + $metastore_database_url = $metastore_database_configs[$metastore_database_type][url] + $metastore_database_driver_class = $metastore_database_configs[$metastore_database_type][driver_class] + + package { 'hive': ensure => latest, - } + } file { "/etc/hive/conf/hive-site.xml": content => template('hadoop-hive/hive-site.xml'), require => Package["hive"], } + + include init-metastore-schema + } + + class client { + + include common + } + + class server { + + include common + + package { 'hive-server2': + ensure => latest, + } + + service { 'hive-server2': + ensure => running, + hasstatus => true, + subscribe => File['/etc/hive/conf/hive-site.xml'], + require => [Package['hive'], Package['hive-server2'], Class['Hadoop-hive::Init-metastore-schema']], + } + } + + class metastore-server { + + include common + + package { 'hive-metastore': + ensure => latest, + } + + service { 'hive-metastore': + ensure => running, + hasstatus => true, + subscribe => File['/etc/hive/conf/hive-site.xml'], + require => [Package['hive'], Package['hive-metastore'], Class['Hadoop-hive::Init-metastore-schema']], + } + } + + class database-connector { + + include common + + case $common::metastore_database_type { + 'mysql': { + contain mysql-connector + } + 'derby': { + # do nothing + } + default: { + fail("$common::metastore_database_type is not supported. Supported database types are ", $common::supported_database_types) + } + } + } + + class mysql-connector { + + include common + + package { 'mysql-connector-java': + ensure => latest + } + + file { '/usr/lib/hive/lib/mysql-connector-java.jar': + ensure => 'link', + target => '/usr/share/java/mysql-connector-java.jar', + require => [Package['mysql-connector-java'], Package['hive']] + } + } + + class init-metastore-schema { + + include common + include database-connector + + exec { 'init hive-metastore schema': + command => "/usr/lib/hive/bin/schematool -dbType $common::metastore_database_type -initSchema -verbose", + require => [Package['hive'], Class['Hadoop-hive::Database-connector'], File['/etc/hive/conf/hive-site.xml']], + subscribe => File['/etc/hive/conf/hive-site.xml'], + logoutput => true, + unless => "/usr/lib/hive/bin/schematool -dbType $common::metastore_database_type -info" + } } } diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml index 2aca237e2..25a9cc5ca 100644 --- a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml @@ -26,38 +26,74 @@ -<% if @hbase_master != "" %> +<% if @hbase_master -%> hbase.master <%= @hbase_master %> http://wiki.apache.org/hadoop/Hive/HBaseIntegration -<% end %> +<% end -%> -<% if @hbase_zookeeper_quorum != "" %> +<% if @hbase_zookeeper_quorum -%> hbase.zookeeper.quorum <%= @hbase_zookeeper_quorum %> http://wiki.apache.org/hadoop/Hive/HBaseIntegration -<% end %> +<% end -%> hive.execution.engine <%= @hive_execution_engine %> +<% end %> +<% if @hdfs_uri -%> + + fs.defaultFS + <%= @hdfs_uri %> + +<% end -%> + +<% if @metastore_server_uris[0] -%> - javax.jdo.option.ConnectionURL - jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true + hive.metastore.uris + <%= @metastore_server_uris.join(",") %> JDBC connect string for a JDBC metastore +<% end -%> + +<% if @metastore_database_url -%> + + javax.jdo.option.ConnectionURL + <%= @metastore_database_url %> + username to use against metastore database + +<% end -%> + +<% if @metastore_database_driver_class -%> + + javax.jdo.option.ConnectionDriverName + <%= @metastore_database_driver_class %> + username to use against metastore database + +<% end -%> + +<% if @metastore_database_user -%> + + javax.jdo.option.ConnectionUserName + <%= @metastore_database_user %> + username to use against metastore database + +<% end -%> +<% if @metastore_database_password -%> - javax.jdo.option.ConnectionDriverName - org.apache.derby.jdbc.EmbeddedDriver - Driver class name for a JDBC metastore + javax.jdo.option.ConnectionPassword + <%= @metastore_database_password %> + password to use against metastore database +<% end -%> hive.hwi.war.file