Update sample configuration files

The sample files in the configs directory are updated to resolve a variety of issues, including for correctness, improved consistency, and breadth of coverage. Notable highlights: * The Solr server role is moved to worker instances. * The Flume agent role is moved to gateway instances. * The newer Oozie service load balancer configuration properties are in place. * Instance template definitions in Azure sample files are dramatically simplified. * The standard Azure bootstrap script is refactored out into its own separate configuration file which is included in Azure sample files.
cloudera · Apr 26, 2019 · 82c14b5 · 82c14b5
1 parent b0ddd20
commit 82c14b5
Show file tree

Hide file tree

Showing 10 changed files with 1,488 additions and 2,007 deletions.
diff --git a/configs/aws.cdsw.conf b/configs/aws.cdsw.conf
@@ -133,7 +133,7 @@ common-instance-properties {
 
     bootstrapScripts: [ """#!/bin/sh
 
-# Remove any natively installed JDKs, as both Cloudera Manager and Cloudera Altus Director only support Oracle JDKs
+# Remove any natively installed JDKs
 yum remove --assumeyes *openjdk*
 
 # Install JDK 1.8 required by Spark 2
@@ -209,7 +209,7 @@ cloudera-manager {
         #
 
         CLOUDERA_MANAGER {
-            enable_api_debug: true
+            # enable_api_debug: true
             custom_banner_html: "Managed by Cloudera Altus Director"
         }
     }
@@ -248,6 +248,9 @@ cluster {
     #
     # Optional override of CDH parcel repositories
     #
+    # This defaults to the Cloudera Enterprise release corresponding to the
+    # Altus Director version.
+    #
 
     parcelRepositories: ["https://archive.cloudera.com/cdh5/parcels/5.15/",
                          "https://archive.cloudera.com/cdsw1/1.4.2/parcels/",

diff --git a/configs/aws.ha.reference.conf b/configs/aws.ha.reference.conf
@@ -476,7 +476,6 @@ cloudera-manager {
   #
   #     NAVIGATOR { ... }
   #
-  #     # Added in Cloudera Manager 5.2+
   #     NAVIGATORMETASERVER { ... }
   # }
 
@@ -524,7 +523,6 @@ cloudera-manager {
       usernamePrefix: navu
     }
 
-    # Added in Cloudera Manager 5.2+
     NAVIGATORMETASERVER {
       name: navmt
       databaseServerName: rds-mysql-prod1
@@ -574,6 +572,9 @@ cluster {
   #
   # Optional override of CDH parcel repositories
   #
+  # This defaults to the Cloudera Enterprise release corresponding to the Altus
+  # Director version.
+  #
 
   # parcelRepositories: ["https://archive.cloudera.com/cdh6/6.2/parcels/"]
 
@@ -605,6 +606,9 @@ cluster {
   # Configuration keys containing special characters (e.g., '.', ':') must be enclosed in double
   # quotes.
   #
+  # Configuration properties for CDH roles and services are documented at
+  # https://www.cloudera.com/documentation/enterprise/6/properties/6.1/topics/cm_props_cdh610.html
+  #
 
   configs {
     # HDFS fencing should be set to true for HA configurations
@@ -879,11 +883,14 @@ cluster {
 
   workers {
     count: 9
+
     #
     # Minimum number of instances required for this instance group.
     # Altus Director will fail bootstrap of a cluster if minCount number of instances are not
     # available in this cloud environment.
+    # If minCount is not specified then minCount is set to count.
     #
+
     minCount: 5
 
     instance: ${instances.d22x} {

diff --git a/configs/aws.reference.conf b/configs/aws.reference.conf
@@ -471,14 +471,60 @@ databaseServers {
   #
 
   rds-mysql-prod1 {
+
+    #
+    # The type of database to use. Allowed values are:
+    #   - mysql
+    #
+
     type: mysql
+
+    #
+    # The root user Cloudera Altus Director will use.
+    #
+
     user: dbrootuser-REPLACEME
+
+    #
+    # The password for the database user.
+    #
+
     password: dbrootpassword-REPLACEME
+
+    #
+    # The instance class of the database server.
+    #
+
     instanceClass: dbinstanceClass-REPLACEME
+
+    #
+    # The DB subnet group for the database server.
+    #
+
     dbSubnetGroupName: dbsubnet-REPLACE-ME
+
+    #
+    # A comma separated list of security group IDs for the database server. The Cloudera Manager
+    # and cluster instances should be permitted by these groups to access the database server.
+    #
+
     vpcSecurityGroupIds: sg-REPLACE-ME
+
+    #
+    # The amount of storage, in GB, to allocate.
+    #
     allocatedStorage: 10
+
+    #
+    # The version of the database engine, e.g., MySQL.
+    #
+
     engineVersion: 5.7.24
+
+    #
+    # Tags for the database server instance
+    #
+
     tags {
       owner: ${?USER}
     }
@@ -520,7 +566,7 @@ cloudera-manager {
   instance: ${instances.m42x} {
     #
     # The properties inherited from the provider section, common-instance-properties or
-    # instance template can be overriden for the Cloudera Manager instance.
+    # instance template can be overridden for the Cloudera Manager instance.
     # These properties include:
     #
 
@@ -652,20 +698,20 @@ cloudera-manager {
   #    Principal@YOUR.KDC.REALM
   #
 
-  # krbAdminUsername: "REPLACE-ME"
+  # krbAdminUsername: "krbAdminUsername-REPLACE-ME"
 
   #
   # The password for the administrative Kerberos account.
   #
 
-  # krbAdminPassword: "REPLACE-ME"
+  # krbAdminPassword: "krbAdminPassword-REPLACE-ME"
 
   #
   # Optional database configuration for Cloudera Manager
   #
   # There are three mutually exclusive options for database usage in Cloudera Altus Director.
   # 1. With no configuration, databases in the Cloudera Manager embedded PostgreSQL database server
-  #    will be used.
+  #    will be used. This option is NOT supported for production use.
   # 2. Alternatively, existing external databases can be used.
   # 3. Finally, databases can be created on the fly on existing external database servers.
   #
@@ -696,20 +742,19 @@ cloudera-manager {
   #
   #     NAVIGATOR { ... }
   #
-  #     # Available in Cloudera Manager 5.2+
   #     NAVIGATORMETASERVER { ... }
   # }
 
   #
   # (Option 3) Optional configuration for creating external databases on the fly
   #
-  # When a database is created on the fly, Director generates a random database name using the
+  # When a database is created on the fly, Altus Director generates a random database name using the
   # specified database name prefix, a random username based on the specified username prefix, and a
-  # random password. The password is stored by Director and made available to the service that uses the database. If
-  # multiple services reference the same external database server, Director will create a database for each.
+  # random password. The password is stored by Altus Director and made available to the service that uses the database.
+  # If multiple services reference the same external database server, Altus Director will create a database for each.
   #
   # MySQL limits usernames to sixteen characters. Therefore, limit usernamePrefix values for databases on MySQL to
-  # seven characters; the remaining nine characters are used by the randomized suffix generated by Director.
+  # seven characters; the remaining nine characters are used by the randomized suffix generated by Altus Director.
   #
   # Note that the databaseServerName must correspond to an external database server named above, which
   # can be the provisioned RDS instance or another existing database server.
@@ -744,7 +789,6 @@ cloudera-manager {
       usernamePrefix: navu
     }
 
-    # Available in Cloudera Manager 5.2+
     NAVIGATORMETASERVER {
       name: navmst
       databaseServerName: rds-mysql-prod1
@@ -788,7 +832,7 @@ cloudera-manager {
     #
 
     CLOUDERA_MANAGER {
-      enable_api_debug: true
+      # enable_api_debug: true
       custom_banner_html: "Managed by Cloudera Altus Director"
 
       #
@@ -806,23 +850,23 @@ cloudera-manager {
       # The KDC host name or IP address.
       #
 
-      # KDC_HOST: "REPLACE-ME"
+      # KDC_HOST: "KDC_HOST-REPLACE-ME"
 
       #
       # The security realm that your KDC uses. This will be of the format of a fully
       # qualified domain name:
       #    YOUR.KDC.REALM
       #
 
-      # SECURITY_REALM: "REPLACE-ME"
+      # SECURITY_REALM: "SECURITY_REALM-REPLACE-ME"
 
       #
       # The Active Directory KDC domain. Only applicable to Active Directory KDCs. This
       # will be in the format of an X.500 Directory Specification:
       #    DC=domain,DC=example,DC=com
       #
 
-      # AD_KDC_DOMAIN: "REPLACE-ME"
+      # AD_KDC_DOMAIN: "AD_KDC_DOMAIN-REPLACE-ME"
 
       #
       # Allow Cloudera Manager to deploy Kerberos configurations to hosts. This should
@@ -884,7 +928,6 @@ cloudera-manager {
     #
     # NAVIGATOR { ... }
     #
-    # # Available in Cloudera Manager 5.2+
     # NAVIGATORMETASERVER { ... }
     #
     # # Configuration properties for all hosts
@@ -921,7 +964,7 @@ cloudera-manager {
 
 echo 'Hello World!'
 exit 0
-    """,
+""",
     {
       id: depPostCreateScript2,
       env {
@@ -933,7 +976,7 @@ exit 0
     """#!/usr/bin/python
 
 print 'Hello again!'
-    """]
+"""]
 
   #
   # More complex scripts can be supplied via local filesystem paths. They will
@@ -1041,17 +1084,20 @@ cluster {
   #
   # Optional override of CDH parcel repositories
   #
+  # This defaults to the Cloudera Enterprise release corresponding to the Altus
+  # Director version.
+  #
 
   # parcelRepositories: ["https://archive.cloudera.com/cdh6/6.2/parcels/"]
 
   #
   # Services to include in the cluster
   #
-  # NOTE: Sentry is only supported in Cloudera Manager versions 5.1 onward.
   # NOTE: On CM 5.9+ Sentry and Kafka 2.0 can't coexist in the same cluster.
   #       If this is needed, use CM 5.8 repository and parcels, or use Kafka
   #       2.1 or higher.
   #
+
   services: [
     HDFS,
     YARN,
@@ -1306,6 +1352,7 @@ cluster {
     # Minimum number of instances required for this instance group.
     # Altus Director will fail bootstrap of a cluster if minCount number of instances are not
     # available in this cloud environment.
+    # If minCount is not specified then minCount is set to count.
     #
     minCount: 5
 
@@ -1503,7 +1550,7 @@ cluster {
 
 echo 'Hello World!'
 exit 0
-    """,
+""",
     {
       id: cluInstancePostCreateScript2,
       env {
@@ -1515,7 +1562,7 @@ exit 0
     """#!/usr/bin/python
 
 print 'Hello again!'
-    """]
+"""]
 
   #
   # More complex scripts can be supplied via local filesystem paths. They will
@@ -1563,7 +1610,7 @@ print 'Hello again!'
 
 echo 'Hello World!'
 exit 0
-    """,
+""",
     {
       id: cluPostCreateScript2,
       env {
@@ -1575,7 +1622,7 @@ exit 0
     """#!/usr/bin/python
 
 print 'Hello again!'
-    """]
+"""]
 
   #
   # More complex scripts can be supplied via local filesystem paths. They will
@@ -1622,9 +1669,9 @@ print 'Hello again!'
 
 echo 'Goodbye World!'
 exit 0
-    """,
+""",
     {
-      id: cluPostCreateScript2,
+      id: cluPreTerminateScript2,
       env {
         KEY1: VALUE1
         KEY2: VALUE2
@@ -1634,7 +1681,7 @@ exit 0
     """#!/usr/bin/python
 
 print 'Goodbye again!'
-        """]
+"""]
 
   #
   # More complex scripts can be supplied via local filesystem paths. They will

diff --git a/configs/aws.simple_setup.conf b/configs/aws.simple_setup.conf
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 #
-# Sample simple AWS Cloudera Altus Director configuration file
+# Sample Cloudera Altus Director configuration file for simple setup in AWS
 #
 # This is a template for use in simple cluster setup for an AWS cluster.
 #
@@ -158,7 +158,7 @@ workerInstanceType: m4.REPLACE-ME
 # OPTIONAL: Operating system for all instances. Must match with instanceImage
 # if specified.
 #
-# Supported for AWS: centos6.7, centos7.2, centos7.4, centos7.5, rhel6.7, rhel7.2, rhel7.4
+# Supported for AWS: centos6.7, centos7.2, centos7.4, centos7.5, centos7.6, rhel6.7, rhel7.2, rhel7.4, rhel7.5, rhel7.6
 # Default: centos7.5
 #
 # os: centos7.5