METRON-1844: Allow for LDAP to be used for authentication and roles

dependencies_with_url.csv

@@ -337,6 +337,8 @@ org.springframework:spring-web:jar:5.0.5.RELEASE:compile,The Apache Software Lic
 org.springframework:spring-webmvc:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework
 org.springframework:spring-webmvc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
 org.springframework:spring-webmvc:jar:5.0.5.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework
+org.springframework.ldap:spring-ldap-core:jar:2.3.2.RELEASE:compile,ASLv2,https://spring.io/projects/spring-ldap
+org.springframework.security:spring-security-ldap:jar:5.1.1.RELEASE:compile,ASLv2,https://spring.io/projects/spring-security
 org.tukaani:xz:jar:1.0:compile,Public Domain,http://tukaani.org/xz/java.html
 org.xerial.snappy:snappy-java:jar:1.0.4.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/snappy-java/
 org.xerial.snappy:snappy-java:jar:1.1.1.7:compile,The Apache Software License, Version 2.0,https://github.com/xerial/snappy-java

metron-deployment/development/README.md

@@ -27,3 +27,37 @@ This directory contains environments useful for Metron developers.  These enviro
 ## Vagrant Cachier recommendations
 
 The development boxes are designed to be spun up and destroyed on a regular basis as part of the development cycle. In order to avoid the overhead of re-downloading many of the heavy platform dependencies, Vagrant can use the [vagrant-cachier](http://fgrehm.viewdocs.io/vagrant-cachier/) plugin to store package caches between builds. If the plugin has been installed to your vagrant it will be used, and packages will be cached in ~/.vagrant/cache.
+
+## Knox Demo LDAP
+
+The development environment can be set up to authenticate against Knox's demo LDAP.
+
+A couple notes
+* A custom LDIF file is used to setup users. This is to get the roles and passwords setup correctly.
+* The demo LDAP uses plaintext passwords with no encryption prefix (e.g. {SSHA}).
+* You may need or want to shut down any or all of the topologies. This is optional, but clears some room
+
+To setup this up, start full dev.
+* In Ambari, add the Knox service (Actions -> +Add Service).  Accept all defaults and let it install. The configs that will be set how we need by default are:
+  * LDAP URL = ldap://localhost:33389
+  * User dn pattern = uid={0},ou=people,dc=hadoop,dc=apache,dc=org
+  * LDAP user searchbase = ou=people,dc=hadoop,dc=apache,dc=org
+  * Group Search Base = ou=groups,dc=hadoop,dc=apache,dc=org
+  * Group Search Filter = member={0}
+  * User Base DN = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+  * User Search Filter is empty
+  * User password attribute = userPassword
+  * LDAP group role attribute = cn
+  * Bind User = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+  * LDAP Truststore is empty
+  * LDAP Truststore Password is empty
+
+* In the Knox configuration, go to "Advanced users-ldif". We have a custom ldif file "knox-demo-ldap.ldif" in "metron-deployment/development" that contains a customized variant of the users and groups defined here. Replace the default ldif configuration with the contents of "knox-demo-ldap.ldif"
+* Start the Demo LDAP (In Knox, "Service Actions -> Start Demo LDAP)
+* In Metron's configs, we're going to make two changes
+  * Set "LDAP Enabled" to "On"
+  * In Security, set "Bind user password" to match the admin user's password from the ldif file (admin-password).
+* Restart the REST application
+
+Now, when you go to Swagger or the UIs, you should be able to give a user and password.
+"admin" will have the roles ROLE_ADMIN and ROLE_USER, which can be verified via the "/whoami/roles" endpoint in Swagger. Similarly, there is a user "sam" that only has ROLE_USER. A third user, "tom" has neither role.

metron-deployment/development/knox-demo-ldap.ldif

@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 1
+
+# Please replace with site specific values
+dn: dc=hadoop,dc=apache,dc=org
+objectclass: organization
+objectclass: dcObject
+o: Hadoop
+dc: hadoop
+
+# Entry for a sample people container
+# Please replace with site specific values
+dn: ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:organizationalUnit
+ou: people
+
+# Entry for a sample end user
+# Please replace with site specific values
+dn: uid=guest,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: Guest
+sn: User
+uid: guest
+userPassword:guest-password
+
+
+# entry for sample user admin
+dn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: Admin
+sn: Admin
+uid: admin
+userPassword:admin-password
+
+# entry for sample user sam
+dn: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: sam
+sn: sam
+uid: sam
+userPassword:sam-password
+
+# entry for sample user tom
+dn: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+objectclass:inetOrgPerson
+cn: tom
+sn: tom
+uid: tom
+userPassword:tom-password
+
+# create FIRST Level groups branch
+dn: ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass:organizationalUnit
+ou: groups
+description: generic groups branch
+
+# create the admin group under groups
+dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: admin
+description:admin group
+member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+
+# create the user group under groups
+dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: user
+description: user group
+member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
+member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org

metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml

@@ -35,32 +35,32 @@
     </property>
     <property>
         <name>metron_spring_profiles_active</name>
-        <description>Active Spring profiles</description>
+        <description>Active Spring profiles. 'ldap' is used to enable authentication via LDAP.</description>
         <display-name>Active Spring profiles</display-name>
         <value/>
         <value-attributes>
             <empty-value-valid>true</empty-value-valid>
         </value-attributes>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_driver</name>
         <value></value>
         <description>Class name of the JDBC Driver used by Metron</description>
         <display-name>Metron JDBC Driver</display-name>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_url</name>
         <value></value>
         <description>JDBC Connection URL used by Metron</description>
         <display-name>Metron JDBC URL</display-name>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_username</name>
         <value></value>
         <description>Metron JDBC Username</description>
         <display-name>Metron JDBC username</display-name>
     </property>
-    <property require-input="true">
+    <property>
         <name>metron_jdbc_password</name>
         <value></value>
         <property-type>PASSWORD</property-type>

metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml

@@ -0,0 +1,186 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration supports_final="true">
+  <property>
+    <name>metron.ldap.enabled</name>
+    <display-name>LDAP Enabled</display-name>
+    <value>false</value>
+    <description>Enable LDAP for Authentication</description>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>true</value>
+          <label>On</label>
+        </entry>
+        <entry>
+          <value>false</value>
+          <label>Off</label>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.url</name>
+    <display-name>LDAP URL</display-name>
+    <value>ldap://localhost:33389</value>
+    <description>LDAP Server URL</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.dnpattern</name>
+    <value>uid={0},ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <display-name>User dn pattern</display-name>
+    <description>LDAP user DN</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.searchbase</name>
+    <display-name>User Search Base</display-name>
+    <value>ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <description>LDAP user searchbase</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.group.searchbase</name>
+    <display-name>Group Search Base</display-name>
+    <value>ou=groups,dc=hadoop,dc=apache,dc=org</value>
+    <description>LDAP group searchbase</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.group.searchfilter</name>
+    <display-name>Group Search Filter</display-name>
+    <value>member={0}</value>
+    <description>LDAP group search filter</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.basedn</name>
+    <display-name>User Base DN</display-name>
+    <value>uid=admin,ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <description>LDAP User Base DN</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.searchfilter</name>
+    <display-name>User Search Filter</display-name>
+    <value></value>
+    <description>Search filter used for Bind Authentication</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.user.password</name>
+    <value>userPassword</value>
+    <display-name>User password attribute</display-name>
+    <description>LDAP attribute for the user password</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.group.roleattribute</name>
+    <display-name>LDAP group role attribute</display-name>
+    <value>cn</value>
+    <description>The LDAP group attribute to be used for determining roles</description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.bind.dn</name>
+    <display-name>Bind User</display-name>
+    <value>uid=admin,ou=people,dc=hadoop,dc=apache,dc=org</value>
+    <description>Full distinguished name (DN), of an LDAP user account that has privileges to search for users. </description>
+    <on-ambari-upgrade add="true"/>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+  <property>
+    <name>metron.ldap.bind.password</name>
+    <display-name>Bind User Password</display-name>
+    <value></value>
+    <property-type>PASSWORD</property-type>
+    <description>Password for the account that can search for users</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+  <property>
+    <name>metron.ldap.ssl.truststore</name>
+    <display-name>LDAP Truststore</display-name>
+    <value></value>
+    <description>Path of truststore with SSL certs for LDAP</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>metron.ldap.ssl.truststore.password</name>
+    <display-name>LDAP Truststore Password</display-name>
+    <value></value>
+    <property-type>PASSWORD</property-type>
+    <description>Password for the truststore with SSL certs for LDAP</description>
+    <value-attributes>
+      <overridable>false</overridable>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+</configuration>

metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml

@@ -322,6 +322,7 @@
             <scriptType>PYTHON</scriptType>
           </commandScript>
           <configuration-dependencies>
+            <config-type>metron-security-env</config-type>
             <config-type>metron-indexing-env</config-type>
             <config-type>metron-rest-env</config-type>
             <config-type>metron-pcap-env</config-type>
@@ -371,6 +372,7 @@
             <scriptType>PYTHON</scriptType>
           </commandScript>
             <configuration-dependencies>
+                <config-type>metron-security-env</config-type>
                 <config-type>metron-rest-env</config-type>
                 <config-type>metron-management-ui-env</config-type>
             </configuration-dependencies>
@@ -397,6 +399,7 @@
             <scriptType>PYTHON</scriptType>
           </commandScript>
           <configuration-dependencies>
+            <config-type>metron-security-env</config-type>
             <config-type>metron-rest-env</config-type>
             <config-type>metron-alerts-ui-env</config-type>
           </configuration-dependencies>