HBASE-25715 Add a basic CompactionServer which only keep heartbeat with HMaster

hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecurityInfo.java

@@ -24,6 +24,7 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.TokenIdentifier.Kind;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.CompactionServerStatusProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos;
@@ -51,6 +52,8 @@ public class SecurityInfo {
         new SecurityInfo(SecurityConstants.MASTER_KRB_PRINCIPAL, Kind.HBASE_AUTH_TOKEN));
     infos.put(MasterProtos.ClientMetaService.getDescriptor().getName(),
         new SecurityInfo(SecurityConstants.MASTER_KRB_PRINCIPAL, Kind.HBASE_AUTH_TOKEN));
+    infos.put(CompactionServerStatusProtos.CompactionServerStatusService.getDescriptor().getName(),
+      new SecurityInfo(SecurityConstants.MASTER_KRB_PRINCIPAL, Kind.HBASE_AUTH_TOKEN));
     // NOTE: IF ADDING A NEW SERVICE, BE SURE TO UPDATE HBasePolicyProvider ALSO ELSE
     // new Service will not be found when all is Kerberized!!!!
   }

hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java

@@ -305,6 +305,29 @@ public enum OperationStatusCode {
   public static final String REGIONSERVER_INFO_PORT_AUTO =
       REGIONSERVER_INFO_PORT + ".auto";
 
+
+  /** Parameter name for port compaction server listens on. */
+  public static final String COMPACTION_SERVER_PORT = "hbase.compaction.server.port";
+
+  /** Default port compaction server listens on. */
+  public static final int DEFAULT_COMPACTION_SERVER_PORT = 16040;
+
+  /** default port for compaction server web api */
+  public static final int DEFAULT_COMPACTION_SERVER_INFOPORT = 16050;
+
+  /** A configuration key for compaction server info port */
+  public static final String COMPACTION_SERVER_INFO_PORT =
+    "hbase.compaction.server.info.port";
+
+  /** A flag that enables automatic selection of compaction server info port */
+  public static final String COMPACTION_SERVER_INFO_PORT_AUTO =
+    COMPACTION_SERVER_INFO_PORT + ".auto";
+
+  /** Parameter name for what compaction server implementation to use. */
+  public static final String COMPACTION_SERVER_IMPL= "hbase.compaction.server.impl";
+
+  public static final String COMPACTION_SERVER_MSG_INTERVAL = "hbase.compaction.server.msginterval";
+
   /** Parameter name for what region server implementation to use. */
   public static final String REGION_SERVER_IMPL= "hbase.regionserver.impl";
 

hbase-common/src/main/java/org/apache/hadoop/hbase/security/SecurityConstants.java

@@ -34,7 +34,10 @@ public final class SecurityConstants {
   public static final String MASTER_KRB_KEYTAB_FILE = "hbase.master.keytab.file";
   public static final String REGIONSERVER_KRB_PRINCIPAL = "hbase.regionserver.kerberos.principal";
   public static final String REGIONSERVER_KRB_KEYTAB_FILE = "hbase.regionserver.keytab.file";
-
+  public static final String COMPACTION_SERVER_KRB_PRINCIPAL =
+      "hbase.compaction.server.kerberos.principal";
+  public static final String COMPACTION_SERVER_KRB_KEYTAB_FILE =
+      "hbase.compaction.server.keytab.file";
   /**
    * This config is for experts: don't set its value unless you really know what you are doing.
    * When set to true, HBase client using SASL Kerberos will skip reverse DNS lookup and use provided

hbase-common/src/main/java/org/apache/hadoop/hbase/util/DNS.java

@@ -38,7 +38,8 @@ public final class DNS {
   public static final String UNSAFE_RS_HOSTNAME_KEY = "hbase.unsafe.regionserver.hostname";
   @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
   public static final String MASTER_HOSTNAME_KEY = "hbase.master.hostname";
-
+  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+  public static final String UNSAFE_CS_HOSTNAME_KEY = "hbase.unsafe.compaction.server.hostname";
   private static boolean HAS_NEW_DNS_GET_DEFAULT_HOST_API;
   private static Method GET_DEFAULT_HOST_METHOD;
 
@@ -64,8 +65,8 @@ public final class DNS {
 
   public enum ServerType {
     MASTER("master"),
-    REGIONSERVER("regionserver");
-
+    REGIONSERVER("regionserver"),
+    COMPACTIONSERVER("compactionserver");
     private String name;
     ServerType(String name) {
       this.name = name;
@@ -118,6 +119,9 @@ public static String getHostname(@NonNull Configuration conf, @NonNull ServerTyp
       case REGIONSERVER:
         hostname = conf.get(UNSAFE_RS_HOSTNAME_KEY);
         break;
+      case COMPACTIONSERVER:
+        hostname = conf.get(UNSAFE_CS_HOSTNAME_KEY);
+        break;
       default:
         hostname = null;
     }

hbase-protocol-shaded/src/main/protobuf/server/CompactionServerStatus.proto

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+syntax = "proto2";
+
+// This file contains protocol buffers that are used for CompactionServerStatusProtocol.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated";
+option java_outer_classname = "CompactionServerStatusProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "server/ClusterStatus.proto";
+import "server/ErrorHandling.proto";
+
+message CompactionServerReportRequest {
+  required ServerName server = 1;
+}
+
+message CompactionServerReportResponse {
+}
+
+service CompactionServerStatusService {
+  /** Called to report the load the CompactionServer is under. */
+  rpc CompactionServerReport(CompactionServerReportRequest)
+    returns(CompactionServerReportResponse);
+}

hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/CompactionServerListTmpl.jamon

@@ -0,0 +1,159 @@
+<%doc>
+Copyright The Apache Software Foundation
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ </%doc>
+
+<%args>
+List<ServerName> servers = null;
+HMaster master;
+</%args>
+
+<%import>
+        java.util.*;
+        org.apache.hadoop.hbase.master.HMaster;
+        org.apache.hadoop.hbase.procedure2.util.StringUtils;
+        org.apache.hadoop.hbase.replication.ReplicationLoadSource;
+        org.apache.hadoop.hbase.RegionMetrics;
+        org.apache.hadoop.hbase.ServerMetrics;
+        org.apache.hadoop.hbase.ServerName;
+        org.apache.hadoop.hbase.Size;
+        org.apache.hadoop.hbase.util.VersionInfo;
+        org.apache.hadoop.hbase.util.Pair;
+        org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
+</%import>
+
+<%if (servers != null && servers.size() > 0)%>
+
+<%java>
+ServerName [] serverNames = servers.toArray(new ServerName[servers.size()]);
+Arrays.sort(serverNames);
+</%java>
+
+<div class="tabbable">
+    <ul class="nav nav-pills">
+        <li class="active"><a href="#tab_compactionserver_baseStats" data-toggle="tab">Base Stats</a></li>
+    </ul>
+    <div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px solid #ddd;">
+        <div class="tab-pane active" id="tab_compactionserver_baseStats">
+            <& baseStats; serverNames = serverNames; &>
+        </div>
+    </div>
+</div>
+
+</%if>
+
+<%def baseStats>
+<%args>
+        ServerName [] serverNames;
+</%args>
+<table id="baseStatsTable" class="tablesorter table table-striped">
+<thead>
+<tr>
+    <th>ServerName</th>
+    <th>Start time</th>
+    <th>Last contact</th>
+    <th>Version</th>
+</tr>
+</thead>
+<tbody>
+<%java>
+    int totalRegions = 0;
+    int totalRequestsPerSecond = 0;
+    int inconsistentNodeNum = 0;
+    String masterVersion = VersionInfo.getVersion();
+    for (ServerName serverName: serverNames) {
+
+    ServerMetrics sl = master.getServerManager().getLoad(serverName);
+    String version = master.getCompactionServerVersion(serverName);
+    if (!masterVersion.equals(version)) {
+        inconsistentNodeNum ++;
+    }
+
+    double requestsPerSecond = 0.0;
+    int numRegionsOnline = 0;
+    long lastContact = 0;
+
+    if (sl != null) {
+        requestsPerSecond = sl.getRequestCountPerSecond();
+        numRegionsOnline = sl.getRegionMetrics().size();
+        totalRegions += sl.getRegionMetrics().size();
+        totalRequestsPerSecond += sl.getRequestCountPerSecond();
+        lastContact = (System.currentTimeMillis() - sl.getReportTimestamp())/1000;
+    }
+    long startcode = serverName.getStartcode();
+</%java>
+<tr>
+    <td><& serverNameLink; serverName=serverName; &></td>
+    <td><% new Date(startcode) %></td>
+    <td><% TraditionalBinaryPrefix.long2String(lastContact, "s", 1) %></td>
+    <td><% version %></td>
+</tr>
+<%java>
+}
+</%java>
+</tbody>
+<tr><td>Total:<% servers.size() %></td>
+<td></td>
+<td></td>
+<%if inconsistentNodeNum > 0%>
+  <td style="color:red;"><% inconsistentNodeNum %> nodes with inconsistent version</td>
+<%else>
+   <td></td>
+</%if>
+</tr>
+</table>
+</%def>
+
+
+<%def serverNameLink>
+        <%args>
+        ServerName serverName;
+        </%args>
+        <%java>
+        int infoPort = master.getCompactionServerInfoPort(serverName);
+        String url = "//" + serverName.getHostname() + ":" + infoPort + "/cs-status";
+        </%java>
+
+        <%if infoPort > 0%>
+            <a href="<% url %>"><% serverName.getServerName() %></a>
+        <%else>
+            <% serverName.getServerName() %>
+        </%if>
+</%def>
+
+<%def emptyStat>
+        <%args>
+        ServerName serverName;
+        </%args>
+    <tr>
+    <td><& serverNameLink; serverName=serverName; &></td>
+    <td></td>
+    <td></td>
+    <td></td>
+    <td></td>
+    <td></td>
+    <td></td>
+    </tr>
+</%def>
+
+
+
+
+
+
+

hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon

@@ -242,6 +242,12 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
                 <& deadRegionServers &>
             </%if>
         </section>
+        <%if master.getCompactionServerManager() != null %>
+            <section>
+                <h2><a name="compactionservers">Compaction Servers</a></h2>
+                <& CompactionServerListTmpl; master= master;servers = master.getCompactionServerManager().getOnlineServersList() &>
+            </section>
+        </%if>
         <section>
             <& BackupMasterStatusTmpl; master = master &>
         </section>