METRON-1844 Allow for LDAP to be used for authentication and roles (j…

…ustinleet) closes #1246
apache · Nov 2, 2018 · fefbb37 · fefbb37
1 parent d0411f6
commit fefbb37
Show file tree

Hide file tree

Showing 18 changed files with 337 additions and 47 deletions.
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
@@ -337,6 +337,8 @@ org.springframework:spring-web:jar:5.0.5.RELEASE:compile,The Apache Software Lic
 org.springframework:spring-webmvc:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
 org.springframework:spring-webmvc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
 org.springframework:spring-webmvc:jar:5.0.5.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
+org.springframework.ldap:spring-ldap-core:jar:2.3.2.RELEASE:compile,ASLv2,https://spring.io/projects/spring-ldap
+org.springframework.security:spring-security-ldap:jar:5.1.1.RELEASE:compile,ASLv2,https://spring.io/projects/spring-security
 org.tukaani:xz:jar:1.0:compile,Public Domain,http://tukaani.org/xz/java.html
 org.xerial.snappy:snappy-java:jar:1.0.4.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/snappy-java/
 org.xerial.snappy:snappy-java:jar:1.1.1.7:compile,The Apache Software License, Version 2.0,https://github.com/xerial/snappy-java

diff --git a/metron-deployment/development/README.md b/metron-deployment/development/README.md
@@ -27,3 +27,37 @@ This directory contains environments useful for Metron developers.  These enviro
 ## Vagrant Cachier recommendations
 
 The development boxes are designed to be spun up and destroyed on a regular basis as part of the development cycle. In order to avoid the overhead of re-downloading many of the heavy platform dependencies, Vagrant can use the [vagrant-cachier](http://fgrehm.viewdocs.io/vagrant-cachier/) plugin to store package caches between builds. If the plugin has been installed to your vagrant it will be used, and packages will be cached in ~/.vagrant/cache.
+
+## Knox Demo LDAP
+
+The development environment can be set up to authenticate against Knox's demo LDAP.
+
+A couple notes
+* A custom LDIF file is used to setup users. This is to get the roles and passwords setup correctly.
+* The demo LDAP uses plaintext passwords with no encryption prefix (e.g. {SSHA}).
+* You may need or want to shut down any or all of the topologies. This is optional, but clears some room
+
+To setup this up, start full dev.
+* In Ambari, add the Knox service (Actions -> +Add Service).  Accept all defaults and let it install. The configs that will be set how we need by default are:
+  * LDAP URL = ldap://localhost:33389
+  * User dn pattern = uid={0},ou=people,dc=hadoop,dc=apache,dc=org
+  * LDAP user searchbase = ou=people,dc=hadoop,dc=apache,dc=org
+  * Group Search Base = ou=groups,dc=hadoop,dc=apache,dc=org
+  * Group Search Filter = member={0}
+  * User Base DN = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+  * User Search Filter is empty
+  * User password attribute = userPassword
+  * LDAP group role attribute = cn
+  * Bind User = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+  * LDAP Truststore is empty
+  * LDAP Truststore Password is empty
+
+* In the Knox configuration, go to "Advanced users-ldif". We have a custom ldif file "knox-demo-ldap.ldif" in "metron-deployment/development" that contains a customized variant of the users and groups defined here. Replace the default ldif configuration with the contents of "knox-demo-ldap.ldif"
+* Start the Demo LDAP (In Knox, "Service Actions -> Start Demo LDAP)
+* In Metron's configs, we're going to make two changes
+  * Set "LDAP Enabled" to "On"
+  * In Security, set "Bind user password" to match the admin user's password from the ldif file (admin-password).
+* Restart the REST application
+
+Now, when you go to Swagger or the UIs, you should be able to give a user and password.
+"admin" will have the roles ROLE_ADMIN and ROLE_USER, which can be verified via the "/whoami/roles" endpoint in Swagger. Similarly, there is a user "sam" that only has ROLE_USER. A third user, "tom" has neither role.
diff --git a/metron-deployment/development/knox-demo-ldap.ldif b/metron-deployment/development/knox-demo-ldap.ldif
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 1
+
+# Please replace with site specific values
+dn: dc=hadoop,dc=apache,dc=org
+objectclass: organization
+objectclass: dcObject
+o: Hadoop
+dc: hadoop
+
+# Entry for a sample people container
+# Please replace with site specific values
+dn: ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:organizationalUnit
+ou: people
+
+# Entry for a sample end user
+# Please replace with site specific values
+dn: uid=guest,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: Guest
+sn: User
+uid: guest
+userPassword:guest-password
+
+
+# entry for sample user admin
+dn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: Admin
+sn: Admin
+uid: admin
+userPassword:admin-password
+
+# entry for sample user sam
+dn: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: sam
+sn: sam
+uid: sam
+userPassword:sam-password
+
+# entry for sample user tom
+dn: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: tom
+sn: tom
+uid: tom
+userPassword:tom-password
+
+# create FIRST Level groups branch
+dn: ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:organizationalUnit
+ou: groups
+description: generic groups branch
+
+# create the admin group under groups
+dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: admin
+description:admin group
+member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+
+# create the user group under groups
+dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: user
+description: user group
+member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
+member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
diff --git a/...mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/...mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -35,9 +35,12 @@
     </property>
     <property>
         <name>metron_spring_profiles_active</name>
-        <description>Active Spring profiles. 'jdbc' is the default profiler for authentication. 'ldap' can is also available.</description>
+        <description>Active Spring profiles. 'ldap' is used to enable authentication via LDAP.</description>
         <display-name>Active Spring profiles</display-name>
-        <value>jdbc</value>
+        <value/>
+        <value-attributes>
+            <empty-value-valid>true</empty-value-valid>
+        </value-attributes>
     </property>
     <property>
         <name>metron_jdbc_driver</name>

diff --git a/...k/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml b/...k/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
@@ -18,10 +18,32 @@
   limitations under the License.
 -->
 <configuration supports_final="true">
+  <property>
+    <name>metron.ldap.enabled</name>
+    <display-name>LDAP Enabled</display-name>
+    <value>false</value>
+    <description>Enable LDAP for Authentication</description>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>true</value>
+          <label>On</label>
+        </entry>
+        <entry>
+          <value>false</value>
+          <label>Off</label>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
   <property>
     <name>metron.ldap.url</name>
     <display-name>LDAP URL</display-name>
-    <value>ldap://localhost:33369</value>
+    <value>ldap://localhost:33389</value>
     <description>LDAP Server URL</description>
     <value-attributes>
       <overridable>false</overridable>
@@ -40,9 +62,9 @@
   </property>
   <property>
     <name>metron.ldap.user.searchbase</name>
-    <display-name>Group Search Base</display-name>
+    <display-name>User Search Base</display-name>
     <value>ou=people,dc=hadoop,dc=apache,dc=org</value>
-    <description>LDAP group searchbase</description>
+    <description>LDAP user searchbase</description>
     <value-attributes>
       <overridable>false</overridable>
       <empty-value-valid>true</empty-value-valid>
@@ -105,8 +127,9 @@
   </property>
   <property>
     <name>metron.ldap.group.roleattribute</name>
+    <display-name>LDAP group role attribute</display-name>
     <value>cn</value>
-    <description>LDAP group role attribute</description>
+    <description>The LDAP group attribute to be used for determining roles</description>
     <value-attributes>
       <overridable>false</overridable>
     </value-attributes>
@@ -116,7 +139,7 @@
     <name>metron.ldap.bind.dn</name>
     <display-name>Bind User</display-name>
     <value>uid=admin,ou=people,dc=hadoop,dc=apache,dc=org</value>
-    <description>Full distinguished name (DN), including common name (CN), of an LDAP user account that has privileges to search for users. </description>
+    <description>Full distinguished name (DN), of an LDAP user account that has privileges to search for users. </description>
     <on-ambari-upgrade add="true"/>
     <value-attributes>
       <overridable>false</overridable>
@@ -136,4 +159,28 @@
     <on-ambari-upgrade add="true"/>
   </property>
 
+  <property>
+    <name>metron.ldap.ssl.truststore</name>
+    <display-name>LDAP Truststore</display-name>
+    <value></value>
+    <description>Path of truststore with SSL certs for LDAP</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.ssl.truststore.password</name>
+    <display-name>LDAP Truststore Password</display-name>
+    <value></value>
+    <property-type>PASSWORD</property-type>
+    <description>Password for the truststore with SSL certs for LDAP</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
 </configuration>
diff --git a/...k/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_commands.py b/...k/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_commands.py
@@ -23,7 +23,6 @@
 
 import metron_service
 
-
 # Wrap major operations and functionality in this class
 class AlertsUICommands:
     __params = None

diff --git a/...c/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_commands.py b/...c/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_commands.py
@@ -22,7 +22,6 @@
 from resource_management.core.resources.system import Execute, File
 from resource_management.core.exceptions import ExecutionFailed
 from resource_management.libraries.functions.get_user_call_output import get_user_call_output
-from resource_management.libraries.functions.format import format
 
 import metron_service
 

diff --git a/.../src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/.../src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -58,7 +58,14 @@
 metron_alerts_ui_host = status_params.metron_alerts_ui_host
 metron_alerts_ui_port = status_params.metron_alerts_ui_port
 metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags']
-metron_spring_profiles_active = config['configurations']['metron-rest-env']['metron_spring_profiles_active']
+
+# Construct the profiles as a temp variable first. Only the first time it's set will carry through
+metron_spring_profiles_temp = config['configurations']['metron-rest-env']['metron_spring_profiles_active']
+if config['configurations']['metron-security-env']['metron.ldap.enabled']:
+    metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap'
+else:
+    metron_spring_profiles_active = metron_spring_profiles_temp
+
 metron_jdbc_driver = config['configurations']['metron-rest-env']['metron_jdbc_driver']
 metron_jdbc_url = config['configurations']['metron-rest-env']['metron_jdbc_url']
 metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_username']
@@ -279,7 +286,8 @@
 metron_ldap_group_searchbase = config['configurations']['metron-security-env']['metron.ldap.group.searchbase']
 metron_ldap_group_searchfilter = config['configurations']['metron-security-env']['metron.ldap.group.searchfilter']
 metron_ldap_group_role = config['configurations']['metron-security-env']['metron.ldap.group.roleattribute']
-metron_ldap = metron_ldap_url != ""
+metron_ldap_ssl_truststore = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore']
+metron_ldap_ssl_truststore_password = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore.password']
 
 # Management UI
 metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0]

diff --git a/...-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/...-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -146,11 +146,11 @@ def start_rest_application(self):
           "export HDP_VERSION={hdp_version};"
           "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};"
           "export METRON_LDAP_PASSWORD={metron_ldap_password!p};"
-          "export METRON_LDAP_USER_PASSWORD={metron_ldap_user_password!p};"
+          "export METRON_LDAP_SSL_TRUSTSTORE_PASSWORD={metron_ldap_ssl_truststore_password!p};"
           "{metron_home}/bin/metron-rest.sh;"
           "unset METRON_JDBC_PASSWORD;"
           "unset METRON_LDAP_PASSWORD;"
-          "unset METRON_LDAP_USER_PASSWORD;"
+          "unset METRON_LDAP_SSL_TRUSTSTORE_PASSWORD;"
         ))
 
         Execute(cmd,

diff --git a/...etron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/...etron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -37,10 +37,12 @@ METRON_LDAP_USERDN="{{metron_ldap_userdn}}"
 METRON_LDAP_USER_PATTERN="{{metron_ldap_user_pattern}}"
 METRON_LDAP_USER_DNBASE="{{metron_ldap_user_dnbase}}"
 METRON_LDAP_USER_SEARCHBASE="{{metron_ldap_user_searchbase}}"
+METRON_LDAP_USER_PASSWORD="{{metron_ldap_user_password}}"
 METRON_LDAP_USER_SEARCHFILTER="{{metron_ldap_user_searchfilter}}"
 METRON_LDAP_GROUP_SEARCHBASE="{{metron_ldap_group_searchbase}}"
 METRON_LDAP_GROUP_SEARCHFILTER="{{metron_ldap_group_searchfilter}}"
 METRON_LDAP_GROUP_ROLE="{{metron_ldap_group_role}}"
+METRON_LDAP_SSL_TRUSTSTORE="{{metron_ldap_ssl_truststore}}"
 
 ZOOKEEPER="{{zookeeper_quorum}}"
 BROKERLIST="{{kafka_brokers}}"

diff --git a/...i/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/...i/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -884,6 +884,10 @@
           "config": "metron-alerts-ui-env/metron_alerts_ui_port",
           "subsection-name": "subsection-alerts-ui"
         },
+        {
+          "config": "metron-security-env/metron.ldap.enabled",
+          "subsection-name": "subsection-security-ldap"
+        },
         {
           "config": "metron-security-env/metron.ldap.url",
           "subsection-name": "subsection-security-ldap"
@@ -924,6 +928,14 @@
           "config": "metron-security-env/metron.ldap.group.roleattribute",
           "subsection-name": "subsection-security-ldap"
         },
+        {
+          "config": "metron-security-env/metron.ldap.ssl.truststore",
+          "subsection-name": "subsection-security-ldap"
+        },
+        {
+          "config": "metron-security-env/metron.ldap.ssl.truststore.password",
+          "subsection-name": "subsection-security-ldap"
+        },
         {
           "config": "metron-pcap-env/pcap_topology_workers",
           "subsection-name": "subsection-pcap"
@@ -1607,6 +1619,12 @@
           "type": "text-field"
         }
       },
+      {
+        "config": "metron-security-env/metron.ldap.enabled",
+        "widget": {
+          "type": "toggle"
+        }
+      },
       {
         "config": "metron-security-env/metron.ldap.url",
         "widget": {
@@ -1625,6 +1643,18 @@
           "type": "password"
         }
       },
+      {
+        "config": "metron-security-env/metron.ldap.ssl.truststore",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-security-env/metron.ldap.ssl.truststore.password",
+        "widget": {
+          "type": "password"
+        }
+      },
       {
         "config": "metron-security-env/metron.ldap.user.dnpattern",
         "widget": {