HADOOP-15821. Move YARN Registry to Hadoop Registry.

Contributed by Íñigo Goiri
apache · Oct 19, 2018 · e2a9fa8 · e2a9fa8
1 parent 00254d7
commit e2a9fa8
Show file tree

Hide file tree

Showing 116 changed files with 516 additions and 369 deletions.
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
@@ -128,6 +128,7 @@ run cp -p "${ROOT}/README.txt" .
 # Remaining projects will copy only libraries which are not present already in 'share' directory.
 run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
 run copy "${ROOT}/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${VERSION}" .
+run copy "${ROOT}/hadoop-common-project/hadoop-registry/target/hadoop-registry-${VERSION}" .
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" .
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .
 run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" .

diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-registry-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-registry-dist.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+  <id>hadoop-registry-dist</id>
+  <formats>
+    <format>dir</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>target</directory>
+      <outputDirectory>/share/hadoop/common</outputDirectory>
+      <includes>
+        <include>${project.artifactId}-${project.version}.jar</include>
+      </includes>
+    </fileSet>
+  </fileSets>
+
+  <dependencySets>
+    <dependencySet>
+      <useProjectArtifact>false</useProjectArtifact>
+      <outputDirectory>/share/hadoop/common/lib</outputDirectory>
+    </dependencySet>
+  </dependencySets>
+
+</assembly>
+
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -477,7 +477,7 @@
         </exclusion>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-registry</artifactId>
+          <artifactId>hadoop-registry</artifactId>
         </exclusion>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>

diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -43,6 +43,7 @@ function hadoop_usage
   hadoop_add_subcommand "jnipath" client "prints the java.library.path"
   hadoop_add_subcommand "kerbname" client "show auth_to_local principal conversion"
   hadoop_add_subcommand "key" client "manage keys via the KeyProvider"
+  hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
   hadoop_add_subcommand "trace" client "view and modify Hadoop tracing settings"
   hadoop_add_subcommand "version" client "print the version"
   hadoop_add_subcommand "kdiag" client "Diagnose Kerberos Problems"
@@ -155,6 +156,11 @@ function hadoopcmd_case
     key)
       HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.KeyShell
     ;;
+    registrydns)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter'
+      HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer'
+    ;;
     trace)
       HADOOP_CLASSNAME=org.apache.hadoop.tracing.TraceAdmin
     ;;

diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -437,3 +437,16 @@ esac
 #
 # For example, to limit who can execute the namenode command,
 # export HDFS_NAMENODE_USER=hdfs
+
+
+###
+# Registry DNS specific parameters
+###
+# For privileged registry DNS, user to run as after dropping privileges
+# This will replace the hadoop.id.str Java property in secure mode.
+# export HADOOP_REGISTRYDNS_SECURE_USER=yarn
+
+# Supplemental options for privileged registry DNS
+# By default, Hadoop uses jsvc which needs to know to launch a
+# server jvm.
+# export HADOOP_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"
diff --git a/...c/site/markdown/registry/yarn-registry.md → ...site/markdown/registry/hadoop-registry.md b/...c/site/markdown/registry/yarn-registry.md → ...site/markdown/registry/hadoop-registry.md
@@ -19,7 +19,7 @@
 
 # Introduction and concepts
 
-This document describes a YARN service registry built to address two problems:
+This document describes a Hadoop service registry built to address two problems:
 
 1. How can clients talk to YARN-deployed services and the components which form
 such services?

diff --git a/...-site/src/site/markdown/registry/index.md → ...ommon/src/site/markdown/registry/index.md b/...-site/src/site/markdown/registry/index.md → ...ommon/src/site/markdown/registry/index.md
@@ -15,7 +15,7 @@
    limitations under the License.
 -->
 
-# YARN Service Registry
+# Hadoop Service Registry
 
 The Service registry is a service which can be deployed in a Hadoop cluster
 to allow deployed applications to register themselves and the means of
@@ -24,7 +24,8 @@ and use the binding information to connect with the services's network-accessibl
 endpoints, be they REST, IPC, Web UI, Zookeeper quorum+path or some other protocol.
 Currently, all the registry data is stored in a zookeeper cluster.
 
-* [Architecture](yarn-registry.html)
+* [Architecture](hadoop-registry.html)
 * [Configuration](registry-configuration.html)
-* [Using the YARN Service registry](using-the-yarn-service-registry.html)
+* [Using the Hadoop Service registry](using-the-hadoop-service-registry.html)
 * [Security](registry-security.html)
+* [Registry DNS](registry-dns.html)
diff --git a/...rkdown/registry/registry-configuration.md → ...rkdown/registry/registry-configuration.md b/...rkdown/registry/registry-configuration.md → ...rkdown/registry/registry-configuration.md
@@ -15,12 +15,12 @@
 # Registry Configuration
 
 
-The YARN service registry is built on top of Apache Zookeeper.
+The Hadoop service registry is built on top of Apache Zookeeper.
 It is configured by way of a Hadoop `Configuration` class:
 the instance used to create the service controls the behavior of the client.
 
 This document lists the configuration parameters which control the
-registry client and its deployment in the YARN Resource Manager.
+registry client.
 
 The default values of all these settings are defined in `core-default.xml`.
 The values in this file may not match those listed in this document.

diff --git a/...site/markdown/yarn-service/RegistryDNS.md → ...rc/site/markdown/registry/registry-dns.md b/...site/markdown/yarn-service/RegistryDNS.md → ...rc/site/markdown/registry/registry-dns.md
@@ -15,13 +15,13 @@
 # Registry DNS Server
 
 <!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-The document describes the internals of Registry DNS server. It is based on the [YARN service registry](../registry/index.html) which is backed by a zookeeper cluster.
+The document describes the internals of Registry DNS server. It is based on the [Hadoop service registry](../registry/index.html) which is backed by a zookeeper cluster.
 ## Introduction
 
-The Registry DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
+The Registry DNS Server provides a standard DNS interface to the information posted into the Hadoop Registry by deployed applications. The DNS service serves the following functions:
 
 1. **Exposing existing service-discovery information via DNS** - Information provided in
-the current YARN service registry’s records will be converted into DNS entries, thus
+the current Hadoop service registry’s records will be converted into DNS entries, thus
 allowing users to discover information about YARN applications using standard DNS
 client mechanisms (e.g. a DNS SRV Record specifying the hostname and port
 number for services).
@@ -32,7 +32,7 @@ http://solr-0.solr-service.devuser.yarncluster:8983/solr/admin/collections?actio
 
 ## Service Properties
 
-The existing YARN Service Registry is leveraged as the source of information for the DNS Service.
+The existing Hadoop Service Registry is leveraged as the source of information for the DNS Service.
 
 The following core functions are supported by the DNS-Server:
 
@@ -139,21 +139,21 @@ RegistryDNS service configured as a forwarder).
 By default, the DNS server runs on non-privileged port `5335`. Start the server
 with:
 ```
-yarn --daemon start registrydns
+hadoop --daemon start registrydns
 ```
 
 If the DNS server is configured to use the standard privileged port `53`, the
-environment variables YARN\_REGISTRYDNS\_SECURE\_USER and
-YARN\_REGISTRYDNS\_SECURE\_EXTRA\_OPTS must be uncommented in the yarn-env.sh
+environment variables HADOOP\_REGISTRYDNS\_SECURE\_USER and
+HADOOP\_REGISTRYDNS\_SECURE\_EXTRA\_OPTS must be uncommented in the hadoop-env.sh
 file. The DNS server should then be launched as root and jsvc will be used to
 reduce the privileges of the daemon after the port has been bound.
 
 ## Configuration
-The Registry DNS server reads its configuration properties from the yarn-site.xml file.  The following are the DNS associated configuration properties:
+The Registry DNS server reads its configuration properties from the core-site.xml file.  The following are the DNS associated configuration properties:
 
 | Name | Description |
 | ------------ | ------------- |
-|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [YARN registry](../registry/registry-configuration.html). |
+|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [Hadoop registry](../registry/registry-configuration.html). |
 | hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
 | hadoop.registry.dns.domain-name  | The domain name for Hadoop cluster associated records.  |
 | hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind.  |
@@ -193,8 +193,32 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
   </property>
 
   <property>
-    <description>A comma separated list of hostname:port pairs defining the zookeeper quorum for the YARN registry</description>
+    <description>A comma separated list of hostname:port pairs defining the zookeeper quorum for the Hadoop registry</description>
     <name>hadoop.registry.zk.quorum</name>
     <value>localhost:2181</value>
   </property>
-  ```
+  ```
+
+To configure Registry DNS to serve reverse lookup for `172.17.0.0/24`
+```
+  <property>
+    <description>The network mask associated with the zone IP range. If specified, it is utilized to ascertain the
+    IP range possible and come up with an appropriate reverse zone name.</description>
+    <name>hadoop.registry.dns.zone-mask</name>
+    <value>255.255.255.0</value>
+  </property>
+
+  <property>
+    <description>An indicator of the IP range associated with the cluster containers. The setting is utilized for the
+     generation of the reverse zone name.</description>
+    <name>hadoop.registry.dns.zone-subnet</name>
+    <value>172.17.0.0</value>
+  </property>
+```
+
+## Make your cluster use Registry DNS
+You can edit the `/etc/resolv.conf` to make your system use the registry DNS such as below, where `192.168.154.3` is the ip address of your DNS host. It should appear before any nameservers that would return NXDOMAIN for lookups in the domain used by the cluster.
+```
+nameserver 192.168.154.3
+```
+Alternatively, if you have a corporate DNS in your organization, you can configure zone forwarding so that the Registry DNS resolves hostnames for the domain used by the cluster.
diff --git a/...te/markdown/registry/registry-security.md → ...te/markdown/registry/registry-security.md b/...te/markdown/registry/registry-security.md → ...te/markdown/registry/registry-security.md
diff --git a/...gistry/using-the-yarn-service-registry.md → ...stry/using-the-hadoop-service-registry.md b/...gistry/using-the-yarn-service-registry.md → ...stry/using-the-hadoop-service-registry.md
@@ -15,9 +15,9 @@
    limitations under the License.
 -->
 
-# Using the YARN Service Registry
+# Using the Hadoop Service Registry
 
-The YARN service registry can be used in a numbe of ways :-
+The Hadoop service registry can be used in a number of ways :-
 
 1. To register dynamic YARN-deployed applications with entries that match the
    lifespan of the YARN application.
@@ -26,7 +26,7 @@ The YARN service registry can be used in a numbe of ways :-
    or an individual container.
 1. To look up static or dynamic applications and the mechanisms to communicate
    with them.
-   Those mechanisms can incude: HTTP(S) URLs, Zookeeper paths,
+   Those mechanisms can include: HTTP(S) URLs, Zookeeper paths,
    hostnames and ports and even paths in a Hadoop filesystem to
    configuration data.
 1. On a secure cluster, to verify that a service binding has been published