AMBARI-411. The HBase puppet templates could include some more config…

… knobs (Contributed by Jaimin Jetly)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1353794 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -6,6 +6,9 @@ characters wide.
 
 Release 1.0.0 - unreleased
 
+  AMBARI-411. The HBase puppet templates could include some more config knobs
+  (Jaimin Jetly via yusaku)
+
   AMBARI-577. Document the steps to build and install monitoring RPMs for Ambari 
               (vgogate)
 

hmc/db/hdp_upgrade_to_1.0.1.php

@@ -15,8 +15,58 @@
 # limitations under the License.
 
 //The script takes the first argument as the path of sqlite database
-
 $db = new PDO("sqlite:".$argv[1]);
+
+
+/***
+ * Adding additional parameters on UI front for HBase
+ */
+
+// maps to hstore_compactionthreshold
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hstore_compactionthreshold\", \"3\", \"HBase HStore compaction threshold\", \"If more than this number of HStoreFiles in any one HStore then a compaction is run to rewrite all HStoreFiles files as one.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hfile_blockcache_size
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hfile_blockcache_size\", \"0.25\", \"HFile block cache size \", \"Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. Set to 0 to disable but it's not recommended.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hstorefile_maxsize.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hstorefile_maxsize\", \"1073741824\", \"Maximum HStoreFile Size\", \"If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"bytes\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to regionserver_handlers.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"regionserver_handlers\", \"30\", \"HBase Region Server Handler\", \"Count of RPC Listener instances spun up on RegionServers\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hregion_majorcompaction.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hregion_majorcompaction\", \"86400000\", \"HBase Region Major Compaction\", \"The time between major compactions of all HStoreFiles in a region. Set to 0 to disable automated major compactions.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"ms\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hregion_blockmultiplier.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hregion_blockmultiplier\", \"2\", \"HBase Region Block Multiplier\", \"Block updates if memstore has \"\"Multiplier * HBase Region Memstore Flush Size\"\" bytes. Useful preventing runaway memstore during spikes in update traffic\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hregion_memstoreflushsize.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hregion_memstoreflushsize\", \"134217728\", \"HBase Region Memstore Flush Size\", \"Memstore will be flushed to disk if size of the memstore exceeds this number of bytes.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"bytes\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to client_scannercaching.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"client_scannercaching\", \"100\", \"HBase Client Scanner Caching\", \"Number of rows that will be fetched when calling next on a scanner if it is not served from (local, client) memory. Do not set this value such that the time between invocations is greater than the scanner timeout\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to zookeeper_sessiontimeout.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"zookeeper_sessiontimeout\", \"60000\", \"Zookeeper timeout for HBase Session\", \"HBase passes this to the zk quorum as suggested maximum time for a session\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"ms\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hfile_max_keyvalue_size.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hfile_max_keyvalue_size\", \"10485760\", \"HBase Client Maximum key-value Size\", \"Specifies the combined maximum allowed size of a KeyValue instance. It should be set to a fraction of the maximum region size.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"bytes\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+
+/***
+ * changes the display on UI from "HIVE SERVER to HIVE METASTORE"
+ */
+
 $query = "UPDATE ServiceComponents SET display_name = 'Hive Metastore' WHERE service_name = 'HIVE' AND component_name = 'HIVE_SERVER'"; 
 $db->exec($query) or die(print_r($db->errorInfo(), true));
 ?>

hmc/db/schema.dump

@@ -338,12 +338,43 @@ INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, de
 -- maps to clientPort in gscluster.
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "clientPort", "2181", "Port for running ZK Server", "Port for running ZooKeeper server", "ZOOKEEPER" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
 
+-- maps to hbase_regionserver_heapsize in gscluster in MB.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hbase_regionserver_heapsize", "1024", "HBase Region Servers maximum Java heap size", "Maximum Java heap size for HBase Region Servers (Java option -Xmx)", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"MB" }' );
 
 -- maps to hbase_master_heapsize in gscluster in MB.
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hbase_master_heapsize", "1024", "HBase Master Maximum Java heap size", "Maximum Java heap size for HBase master (Java option -Xmx)", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"MB" }' );
 
--- maps to hbase_regionserver_heapsize in gscluster in MB.
-INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hbase_regionserver_heapsize", "1024", "HBase Region Servers maximum Java heap size", "Maximum Java heap size for HBase Region Servers (Java option -Xmx)", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"MB" }' );
+--maps to hstore_compactionthreshold
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hstore_compactionthreshold", "3", "HBase HStore compaction threshold", "If more than this number of HStoreFiles in any one HStore then a compaction is run to rewrite all HStoreFiles files as one.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+--maps to hfile_blockcache_size
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hfile_blockcache_size", "0.25", "HFile block cache size ", "Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. Set to 0 to disable but it's not recommended.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to hstorefile_maxsize. 
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hstorefile_maxsize", "1073741824", "Maximum HStoreFile Size", "If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"bytes" }' );
+
+-- maps to regionserver_handlers. 
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "regionserver_handlers", "30", "HBase Region Server Handler", "Count of RPC Listener instances spun up on RegionServers", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to hregion_majorcompaction.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hregion_majorcompaction", "86400000", "HBase Region Major Compaction", "The time between major compactions of all HStoreFiles in a region. Set to 0 to disable automated major compactions.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"ms" }' );
+
+-- maps to hregion_blockmultiplier.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hregion_blockmultiplier", "2", "HBase Region Block Multiplier", "Block updates if memstore has ""Multiplier * HBase Region Memstore Flush Size"" bytes. Useful preventing runaway memstore during spikes in update traffic", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to hregion_memstoreflushsize.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hregion_memstoreflushsize", "134217728", "HBase Region Memstore Flush Size", "Memstore will be flushed to disk if size of the memstore exceeds this number of bytes.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"bytes" }' );
+
+-- maps to client_scannercaching.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "client_scannercaching", "100", "HBase Client Scanner Caching", "Number of rows that will be fetched when calling next on a scanner if it is not served from (local, client) memory. Do not set this value such that the time between invocations is greater than the scanner timeout", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to zookeeper_sessiontimeout.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "zookeeper_sessiontimeout", "60000", "Zookeeper timeout for HBase Session", "HBase passes this to the zk quorum as suggested maximum time for a session", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"ms" }' );
+
+-- maps to hfile_max_keyvalue_size.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hfile_max_keyvalue_size", "10485760", "HBase Client Maximum key-value Size", "Specifies the combined maximum allowed size of a KeyValue instance. It should be set to a fraction of the maximum region size.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"bytes" }' );
+
+
 
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "lzo_enabled", "false", "LZO compression", "LZO compression enabled", "MAPREDUCE" , "ONOFF", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text" }' );
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "snappy_enabled", "true", "Snappy compression", "Snappy compression enabled", "MAPREDUCE" , "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );

hmc/puppet/modules/hdp-hbase/manifests/params.pp

@@ -32,11 +32,25 @@
 
   $hfile_blockcache_size = hdp_default("hadoop/hbase-site/hfile_blockcache_size","0.25")
 
+  $hfile_max_keyvalue_size = hdp_default("hadoop/hbase-site/hfile_max_keyvalue_size",10485760)
+
+  $zookeeper_sessiontimeout = hdp_default("hadoop/hbase-site/zookeeper_sessiontimeout",60000)
+
+  $client_scannercaching = hdp_default("hadoop/hbase-site/client_scannercaching",100)
+
   $hstore_blockingstorefiles = hdp_default("hadoop/hbase-site/hstore_blockingstorefiles",7)
 
   $hstore_compactionthreshold = hdp_default("hadoop/hbase-site/hstore_compactionthreshold",3)
 
-  $hstorefile_maxsize = hdp_default("hadoop/hbase-site/hstorefile_maxsize",268435456)
+  $hstorefile_maxsize = hdp_default("hadoop/hbase-site/hstorefile_maxsize",1073741824)
+
+  $hregion_blockmultiplier = hdp_default("hadoop/hbase-site/hregion_blockmultiplier",2)
+
+  $hregion_memstoreflushsize = hdp_default("hadoop/hbase-site/hregion_memstoreflushsize",134217728)
+
+  $regionserver_handlers = hdp_default("hadoop/hbase-site/regionserver_handlers", 30)
+
+  $hregion_majorcompaction = hdp_default("hadoop/hbase-site/hregion_majorcompaction", 86400000)
 
   $preloaded_mastercoprocessor_classes = hdp_default("hadoop/hbase-site/preloaded_mastercoprocessor_classes")
 

hmc/puppet/modules/hdp-hbase/templates/hbase-site.xml.erb

@@ -64,6 +64,22 @@
       updates are blocked and flushes are forced. Defaults to 40% of heap
     </description>
   </property>
+  <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value><%=scope.function_hdp_template_var("regionserver_handlers")%></value>
+    <description>Count of RPC Listener instances spun up on RegionServers.
+    Same property is used by the Master for count of master handlers.
+    Default is 10.
+    </description>
+  </property>
+  <property>
+    <name>hbase.hregion.majorcompaction</name>
+    <value><%=scope.function_hdp_template_var("hregion_majorcompaction")%></value>
+    <description>The time (in miliseconds) between 'major' compactions of all
+    HStoreFiles in a region.  Default: 1 day.
+    Set to 0 to disable automated major compactions.
+    </description>
+  </property>
   <property>
     <name>hbase.regionserver.global.memstore.lowerLimit</name>
     <value><%=scope.function_hdp_template_var("regionserver_memstore_lowerlimit")%></value>
@@ -74,6 +90,26 @@
       memstore limiting.
     </description>
   </property>
+  <property>
+    <name>hbase.hregion.memstore.block.multiplier</name>
+    <value><%=scope.function_hdp_template_var("hregion_blockmultiplier")%></value>
+    <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier
+    time hbase.hregion.flush.size bytes.  Useful preventing
+    runaway memstore during spikes in update traffic.  Without an
+    upper-bound, memstore fills such that when it flushes the
+    resultant flush files take a long time to compact or split, or
+    worse, we OOME
+    </description>
+  </property>
+  <property>
+    <name>hbase.hregion.memstore.flush.size</name>
+    <value><%=scope.function_hdp_template_var("hregion_memstoreflushsize")%></value>
+    <description>
+    Memstore will be flushed to disk if size of the memstore
+    exceeds this number of bytes.  Value is checked by a thread that runs
+    every hbase.server.thread.wakefrequency.
+    </description>
+  </property>
   <property>
     <name>hbase.hregion.memstore.mslab.enabled</name>
     <value><%=scope.function_hdp_template_var("regionserver_memstore_lab")%></value>
@@ -93,6 +129,39 @@
     Default: 1G.
     </description>
   </property>
+  <property>
+    <name>hbase.client.scanner.caching</name>
+    <value><%=scope.function_hdp_template_var("client_scannercaching")%></value>
+    <description>Number of rows that will be fetched when calling next
+    on a scanner if it is not served from (local, client) memory. Higher
+    caching values will enable faster scanners but will eat up more memory
+    and some calls of next may take longer and longer times when the cache is empty.
+    Do not set this value such that the time between invocations is greater
+    than the scanner timeout; i.e. hbase.regionserver.lease.period
+    </description>
+  </property>
+  <property>
+    <name>zookeeper.session.timeout</name>
+    <value><%=scope.function_hdp_template_var("zookeeper_sessiontimeout")%></value>
+    <description>ZooKeeper session timeout.
+      HBase passes this to the zk quorum as suggested maximum time for a
+      session (This setting becomes zookeeper's 'maxSessionTimeout').  See
+      http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+      "The client sends a requested timeout, the server responds with the
+      timeout that it can give the client. " In milliseconds.
+    </description>
+  </property>
+  <property>
+    <name>hbase.client.keyvalue.maxsize</name>
+    <value><%=scope.function_hdp_template_var("hfile_max_keyvalue_size")%></value>
+    <description>Specifies the combined maximum allowed size of a KeyValue
+    instance. This is to set an upper boundary for a single entry saved in a
+    storage file. Since they cannot be split it helps avoiding that a region
+    cannot be split any further because the data is too large. It seems wise
+    to set this to a fraction of the maximum region size. Setting it to zero
+    or less disables the check.
+    </description>
+  </property>
   <property>
     <name>hbase.hstore.compactionThreshold</name>
     <value><%=scope.function_hdp_template_var("hstore_compactionthreshold")%></value>