diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 2e1eedd620..7bc0e9be3d 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -337,6 +337,8 @@ org.springframework:spring-web:jar:5.0.5.RELEASE:compile,The Apache Software Lic org.springframework:spring-webmvc:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework org.springframework:spring-webmvc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-webmvc:jar:5.0.5.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework +org.springframework.ldap:spring-ldap-core:jar:2.3.2.RELEASE:compile,ASLv2,https://spring.io/projects/spring-ldap +org.springframework.security:spring-security-ldap:jar:5.1.1.RELEASE:compile,ASLv2,https://spring.io/projects/spring-security org.tukaani:xz:jar:1.0:compile,Public Domain,http://tukaani.org/xz/java.html org.xerial.snappy:snappy-java:jar:1.0.4.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/snappy-java/ org.xerial.snappy:snappy-java:jar:1.1.1.7:compile,The Apache Software License, Version 2.0,https://github.com/xerial/snappy-java diff --git a/metron-deployment/development/README.md b/metron-deployment/development/README.md index 2a04e5f9f3..b86a5c4c7e 100644 --- a/metron-deployment/development/README.md +++ b/metron-deployment/development/README.md @@ -27,3 +27,37 @@ This directory contains environments useful for Metron developers. These enviro ## Vagrant Cachier recommendations The development boxes are designed to be spun up and destroyed on a regular basis as part of the development cycle. In order to avoid the overhead of re-downloading many of the heavy platform dependencies, Vagrant can use the [vagrant-cachier](http://fgrehm.viewdocs.io/vagrant-cachier/) plugin to store package caches between builds. If the plugin has been installed to your vagrant it will be used, and packages will be cached in ~/.vagrant/cache. + +## Knox Demo LDAP + +The development environment can be set up to authenticate against Knox's demo LDAP. + +A couple notes +* A custom LDIF file is used to setup users. This is to get the roles and passwords setup correctly. +* The demo LDAP uses plaintext passwords with no encryption prefix (e.g. {SSHA}). +* You may need or want to shut down any or all of the topologies. This is optional, but clears some room + +To setup this up, start full dev. +* In Ambari, add the Knox service (Actions -> +Add Service). Accept all defaults and let it install. The configs that will be set how we need by default are: + * LDAP URL = ldap://localhost:33389 + * User dn pattern = uid={0},ou=people,dc=hadoop,dc=apache,dc=org + * LDAP user searchbase = ou=people,dc=hadoop,dc=apache,dc=org + * Group Search Base = ou=groups,dc=hadoop,dc=apache,dc=org + * Group Search Filter = member={0} + * User Base DN = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + * User Search Filter is empty + * User password attribute = userPassword + * LDAP group role attribute = cn + * Bind User = uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + * LDAP Truststore is empty + * LDAP Truststore Password is empty + +* In the Knox configuration, go to "Advanced users-ldif". We have a custom ldif file "knox-demo-ldap.ldif" in "metron-deployment/development" that contains a customized variant of the users and groups defined here. Replace the default ldif configuration with the contents of "knox-demo-ldap.ldif" +* Start the Demo LDAP (In Knox, "Service Actions -> Start Demo LDAP) +* In Metron's configs, we're going to make two changes + * Set "LDAP Enabled" to "On" + * In Security, set "Bind user password" to match the admin user's password from the ldif file (admin-password). +* Restart the REST application + +Now, when you go to Swagger or the UIs, you should be able to give a user and password. +"admin" will have the roles ROLE_ADMIN and ROLE_USER, which can be verified via the "/whoami/roles" endpoint in Swagger. Similarly, there is a user "sam" that only has ROLE_USER. A third user, "tom" has neither role. diff --git a/metron-deployment/development/knox-demo-ldap.ldif b/metron-deployment/development/knox-demo-ldap.ldif new file mode 100644 index 0000000000..3097a6438a --- /dev/null +++ b/metron-deployment/development/knox-demo-ldap.ldif @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 1 + +# Please replace with site specific values +dn: dc=hadoop,dc=apache,dc=org +objectclass: organization +objectclass: dcObject +o: Hadoop +dc: hadoop + +# Entry for a sample people container +# Please replace with site specific values +dn: ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:organizationalUnit +ou: people + +# Entry for a sample end user +# Please replace with site specific values +dn: uid=guest,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: Guest +sn: User +uid: guest +userPassword:guest-password + + +# entry for sample user admin +dn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: Admin +sn: Admin +uid: admin +userPassword:admin-password + +# entry for sample user sam +dn: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: sam +sn: sam +uid: sam +userPassword:sam-password + +# entry for sample user tom +dn: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: tom +sn: tom +uid: tom +userPassword:tom-password + +# create FIRST Level groups branch +dn: ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:organizationalUnit +ou: groups +description: generic groups branch + +# create the admin group under groups +dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: admin +description:admin group +member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + +# create the user group under groups +dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: user +description: user group +member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org +member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index f4b2327d80..55b880f2d6 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -35,32 +35,32 @@ metron_spring_profiles_active - Active Spring profiles + Active Spring profiles. 'ldap' is used to enable authentication via LDAP. Active Spring profiles true - + metron_jdbc_driver Class name of the JDBC Driver used by Metron Metron JDBC Driver - + metron_jdbc_url JDBC Connection URL used by Metron Metron JDBC URL - + metron_jdbc_username Metron JDBC Username Metron JDBC username - + metron_jdbc_password PASSWORD diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml new file mode 100644 index 0000000000..ab1fe6ce2f --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml @@ -0,0 +1,186 @@ + + + + + + metron.ldap.enabled + LDAP Enabled + false + Enable LDAP for Authentication + + value-list + + + true + + + + false + + + + 1 + false + + + + + metron.ldap.url + LDAP URL + ldap://localhost:33389 + LDAP Server URL + + false + + + + + metron.ldap.user.dnpattern + uid={0},ou=people,dc=hadoop,dc=apache,dc=org + User dn pattern + LDAP user DN + + false + + + + + metron.ldap.user.searchbase + User Search Base + ou=people,dc=hadoop,dc=apache,dc=org + LDAP user searchbase + + false + true + + + + + metron.ldap.group.searchbase + Group Search Base + ou=groups,dc=hadoop,dc=apache,dc=org + LDAP group searchbase + + false + true + + + + + metron.ldap.group.searchfilter + Group Search Filter + member={0} + LDAP group search filter + + false + true + + + + + metron.ldap.user.basedn + User Base DN + uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + LDAP User Base DN + + false + true + + + + + metron.ldap.user.searchfilter + User Search Filter + + Search filter used for Bind Authentication + + false + true + + + + + metron.ldap.user.password + userPassword + User password attribute + LDAP attribute for the user password + + false + + + + + metron.ldap.group.roleattribute + LDAP group role attribute + cn + The LDAP group attribute to be used for determining roles + + false + + + + + metron.ldap.bind.dn + Bind User + uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + Full distinguished name (DN), of an LDAP user account that has privileges to search for users. + + + false + true + + + + metron.ldap.bind.password + Bind User Password + + PASSWORD + Password for the account that can search for users + + false + true + + + + + + metron.ldap.ssl.truststore + LDAP Truststore + + Path of truststore with SSL certs for LDAP + + false + true + + + + + metron.ldap.ssl.truststore.password + LDAP Truststore Password + + PASSWORD + Password for the truststore with SSL certs for LDAP + + false + true + + + + + \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index ad1f7a9074..97b574922c 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -322,6 +322,7 @@ PYTHON + metron-security-env metron-indexing-env metron-rest-env metron-pcap-env @@ -371,6 +372,7 @@ PYTHON + metron-security-env metron-rest-env metron-management-ui-env @@ -397,6 +399,7 @@ PYTHON + metron-security-env metron-rest-env metron-alerts-ui-env diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 458a7bedea..75f68fc2bc 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -58,7 +58,14 @@ metron_alerts_ui_host = status_params.metron_alerts_ui_host metron_alerts_ui_port = status_params.metron_alerts_ui_port metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags'] -metron_spring_profiles_active = config['configurations']['metron-rest-env']['metron_spring_profiles_active'] + +# Construct the profiles as a temp variable first. Only the first time it's set will carry through +metron_spring_profiles_temp = config['configurations']['metron-rest-env']['metron_spring_profiles_active'] +if config['configurations']['metron-security-env']['metron.ldap.enabled']: + metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap' +else: + metron_spring_profiles_active = metron_spring_profiles_temp + metron_jdbc_driver = config['configurations']['metron-rest-env']['metron_jdbc_driver'] metron_jdbc_url = config['configurations']['metron-rest-env']['metron_jdbc_url'] metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_username'] @@ -267,6 +274,21 @@ if 'solr-config-env' in config['configurations']: solr_principal_name = solr_principal_name.replace('_HOST', hostname_lowercase) +# LDAP +metron_ldap_url = config['configurations']['metron-security-env']['metron.ldap.url'] +metron_ldap_userdn = config['configurations']['metron-security-env']['metron.ldap.bind.dn'] +metron_ldap_password = config['configurations']['metron-security-env']['metron.ldap.bind.password'] +metron_ldap_user_pattern = config['configurations']['metron-security-env']['metron.ldap.user.dnpattern'] +metron_ldap_user_password = config['configurations']['metron-security-env']['metron.ldap.user.password'] +metron_ldap_user_dnbase = config['configurations']['metron-security-env']['metron.ldap.user.basedn'] +metron_ldap_user_searchbase = config['configurations']['metron-security-env']['metron.ldap.user.searchbase'] +metron_ldap_user_searchfilter = config['configurations']['metron-security-env']['metron.ldap.user.searchfilter'] +metron_ldap_group_searchbase = config['configurations']['metron-security-env']['metron.ldap.group.searchbase'] +metron_ldap_group_searchfilter = config['configurations']['metron-security-env']['metron.ldap.group.searchfilter'] +metron_ldap_group_role = config['configurations']['metron-security-env']['metron.ldap.group.roleattribute'] +metron_ldap_ssl_truststore = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore'] +metron_ldap_ssl_truststore_password = config['configurations']['metron-security-env']['metron.ldap.ssl.truststore.password'] + # Management UI metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0] diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index c410b94a49..bab912977d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -145,8 +145,12 @@ def start_rest_application(self): "export METRON_PID_FILE={pid_file};" "export HDP_VERSION={hdp_version};" "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};" + "export METRON_LDAP_PASSWORD={metron_ldap_password!p};" + "export METRON_LDAP_SSL_TRUSTSTORE_PASSWORD={metron_ldap_ssl_truststore_password!p};" "{metron_home}/bin/metron-rest.sh;" "unset METRON_JDBC_PASSWORD;" + "unset METRON_LDAP_PASSWORD;" + "unset METRON_LDAP_SSL_TRUSTSTORE_PASSWORD;" )) Execute(cmd, diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index a7d01e5b5d..08d4281ca7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -21,6 +21,8 @@ METRON_PID_DIR="{{metron_pid_dir}}" METRON_REST_PORT={{metron_rest_port}} METRON_JVMFLAGS="{{metron_jvm_flags}}" METRON_SPRING_PROFILES_ACTIVE="{{metron_spring_profiles_active}}" + +#JDBC METRON_JDBC_DRIVER="{{metron_jdbc_driver}}" METRON_JDBC_URL="{{metron_jdbc_url}}" METRON_JDBC_USERNAME="{{metron_jdbc_username}}" @@ -28,6 +30,20 @@ METRON_JDBC_PLATFORM="{{metron_jdbc_platform}}" METRON_JDBC_CLIENT_PATH="{{metron_jdbc_client_path}}" METRON_TEMP_GROK_PATH="{{metron_temp_grok_path}}" METRON_SPRING_OPTIONS="{{metron_spring_options}}" + +#LDAP +METRON_LDAP_URL="{{metron_ldap_url}}" +METRON_LDAP_USERDN="{{metron_ldap_userdn}}" +METRON_LDAP_USER_PATTERN="{{metron_ldap_user_pattern}}" +METRON_LDAP_USER_DNBASE="{{metron_ldap_user_dnbase}}" +METRON_LDAP_USER_SEARCHBASE="{{metron_ldap_user_searchbase}}" +METRON_LDAP_USER_PASSWORD="{{metron_ldap_user_password}}" +METRON_LDAP_USER_SEARCHFILTER="{{metron_ldap_user_searchfilter}}" +METRON_LDAP_GROUP_SEARCHBASE="{{metron_ldap_group_searchbase}}" +METRON_LDAP_GROUP_SEARCHFILTER="{{metron_ldap_group_searchfilter}}" +METRON_LDAP_GROUP_ROLE="{{metron_ldap_group_role}}" +METRON_LDAP_SSL_TRUSTSTORE="{{metron_ldap_ssl_truststore}}" + ZOOKEEPER="{{zookeeper_quorum}}" BROKERLIST="{{kafka_brokers}}" HADOOP_CONF_DIR="/etc/hadoop/conf/" diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 7e6c83a756..1d7b6c5a20 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -421,6 +421,35 @@ ] } }, + { + "name": "security", + "display-name": "Security", + "layout": { + "tab-columns": "2", + "tab-rows": "1", + "sections": [ + { + "name": "section-security-ldap", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-security-ldap", + "display-name": "LDAP", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, { "name": "metron-pcap", "display-name": "PCAP", @@ -855,6 +884,58 @@ "config": "metron-alerts-ui-env/metron_alerts_ui_port", "subsection-name": "subsection-alerts-ui" }, + { + "config": "metron-security-env/metron.ldap.enabled", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.url", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.bind.dn", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.bind.password", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.dnpattern", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.password", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.searchbase", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.user.searchfilter", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.group.searchbase", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.group.searchfilter", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.group.roleattribute", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore", + "subsection-name": "subsection-security-ldap" + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore.password", + "subsection-name": "subsection-security-ldap" + }, { "config": "metron-pcap-env/pcap_topology_workers", "subsection-name": "subsection-pcap" @@ -1538,6 +1619,84 @@ "type": "text-field" } }, + { + "config": "metron-security-env/metron.ldap.enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "metron-security-env/metron.ldap.url", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.bind.dn", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.bind.password", + "widget": { + "type": "password" + } + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.ssl.truststore.password", + "widget": { + "type": "password" + } + }, + { + "config": "metron-security-env/metron.ldap.user.dnpattern", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.user.password", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.user.searchbase", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.user.searchfilter", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.group.searchbase", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.group.searchfilter", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-security-env/metron.ldap.group.roleattribute", + "widget": { + "type": "text-field" + } + }, { "config": "metron-pcap-env/pcap_topology_workers", "widget": { diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 7f00cde0dd..f85707f3c9 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -194,6 +194,44 @@ METRON_PRINCIPAL_NAME="metron@EXAMPLE.COM" METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" ``` +### LDAP + +Metron REST can be configured to use LDAP for authentication and roles. Configuration can be performed via Ambari in the "Security" tab. + +Configuration will default to matching Knox's Demo LDAP for convenience. This should only be used for development purposes. Manual instructions for setting up demo LDAP and finalizing configuration (e.g. setting up the user LDIF file) can be found in the [Development README](../../metron-deployment/development/README.md#knox-demo-ldap). + +#### LDAPS +There is configuration to provide a path to a truststore with SSL certificates and provide a password. Users should import certificates as needed to appropriate truststores. An example of doing this is: +``` +keytool -import -alias -file -keystore -storepass +``` + + +#### Roles +Roles used by Metron are ROLE_ADMIN and ROLE_USER. Metron will use a property in a group containing the appropriate role to construct this. + +For example, our ldif file could create this group: +``` +dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: admin +description:admin group +member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org +``` + +If we are using "cn" as our role attribute, Metron will give the "admin" user the role "ROLE_ADMIN". + +Similarly, we could give a user "sam" ROLE_USER with the following group: +``` +dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: user +description: user group +member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org +``` + ## Spring Profiles The REST application comes with a few [Spring Profiles](http://docs.spring.io/autorepo/docs/spring-boot/current/reference/html/boot-features-profiles.html) to aid in testing and development. diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 7fc373bb5a..52b01754b2 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -31,6 +31,8 @@ 1.6.4 2.0.1.RELEASE 1.0.1.RELEASE + 2.3.2.RELEASE + 5.1.1.RELEASE 2.5.0 5.1.40 2.0.4.RELEASE @@ -117,6 +119,16 @@ + + org.springframework.ldap + spring-ldap-core + ${spring.ldap.core.version} + + + org.springframework.security + spring-security-ldap + ${spring.security.ldap.version} + com.googlecode.json-simple json-simple diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml index 84efc01b91..7ab7b7303e 100644 --- a/metron-interface/metron-rest/src/main/config/rest_application.yml +++ b/metron-interface/metron-rest/src/main/config/rest_application.yml @@ -13,14 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -spring: - datasource: - driverClassName: ${METRON_JDBC_DRIVER} - url: ${METRON_JDBC_URL} - username: ${METRON_JDBC_USERNAME} - password: ${METRON_JDBC_PASSWORD} - platform: ${METRON_JDBC_PLATFORM} - continue-on-error: true zookeeper: url: ${ZOOKEEPER} @@ -62,3 +54,27 @@ pcap: page.size: ${PCAP_PAGE_SIZE} yarn.queue: ${PCAP_YARN_QUEUE} finalizer.threadpool.size: ${PCAP_FINALIZER_THREADPOOL_SIZE} + +spring: + datasource: + driverClassName: ${METRON_JDBC_DRIVER} + url: ${METRON_JDBC_URL} + username: ${METRON_JDBC_USERNAME} + password: ${METRON_JDBC_PASSWORD} + platform: ${METRON_JDBC_PLATFORM} + continue-on-error: true + +ldap: + provider: + url: ${METRON_LDAP_URL} + userdn: ${METRON_LDAP_USERDN} + password: ${METRON_LDAP_PASSWORD} + user: + dn.patterns: ${METRON_LDAP_USER_PATTERN} + passwordAttribute: ${METRON_LDAP_USER_PASSWORD} + searchBase: ${METRON_LDAP_USER_SEARCHBASE} + searchFilter: ${METRON_LDAP_USER_SEARCHFILTER} + group: + searchBase: ${METRON_LDAP_GROUP_SEARCHBASE} + searchFilter: ${METRON_LDAP_GROUP_SEARCHFILTER} + roleAttribute: ${METRON_LDAP_GROUP_ROLE} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index 94e8e35b05..80ac2bf3ab 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -23,6 +23,7 @@ public class MetronRestConstants { public static final String DEV_PROFILE = "dev"; public static final String TEST_PROFILE = "test"; + public static final String LDAP_PROFILE = "ldap"; public static final String DOCKER_PROFILE = "docker"; public static final String CSRF_ENABLE_PROFILE = "csrf-enable"; diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java index f84cdfabab..7ca3a46e4b 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java @@ -20,8 +20,14 @@ import static org.apache.metron.rest.MetronRestConstants.SECURITY_ROLE_ADMIN; import static org.apache.metron.rest.MetronRestConstants.SECURITY_ROLE_USER; +import java.util.Arrays; +import java.util.List; +import javax.sql.DataSource; import org.apache.metron.rest.MetronRestConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; @@ -30,7 +36,7 @@ import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -import org.springframework.security.core.userdetails.User; +import org.springframework.security.crypto.password.LdapShaPasswordEncoder; import org.springframework.security.crypto.password.NoOpPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.authentication.logout.HttpStatusReturningLogoutSuccessHandler; @@ -39,19 +45,40 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; -import javax.sql.DataSource; -import java.util.Arrays; -import java.util.List; - @Configuration @EnableWebSecurity @EnableGlobalMethodSecurity(securedEnabled = true) @Controller public class WebSecurityConfig extends WebSecurityConfigurerAdapter { + private static final Logger LOG = LoggerFactory.getLogger(WebSecurityConfig.class); @Autowired private Environment environment; + @Autowired + private DataSource dataSource; + + @Value("${ldap.provider.url}") + private String providerUrl; + @Value("${ldap.provider.userdn}") + private String providerUserDn; + @Value("${ldap.provider.password}") + private String providerPassword; + @Value("${ldap.user.dn.patterns}") + private String userDnPatterns; + @Value("${ldap.user.passwordAttribute}") + private String passwordAttribute; + @Value("${ldap.user.searchBase}") + private String userSearchBase; + @Value("${ldap.user.searchFilter}") + private String userSearchFilter; + @Value("${ldap.group.searchBase}") + private String groupSearchBase; + @Value("${ldap.group.roleAttribute}") + private String groupRoleAttribute; + @Value("${ldap.group.searchFilter}") + private String groupSearchFilter; + @RequestMapping(value = {"/login", "/logout", "/sensors", "/sensors*/**"}, method = RequestMethod.GET) public String handleNGRequests() { return "forward:/index.html"; @@ -83,19 +110,35 @@ protected void configure(HttpSecurity http) throws Exception { } } - @Autowired - private DataSource dataSource; - @Autowired public void configureJdbc(AuthenticationManagerBuilder auth) throws Exception { + // Note that we can switch profiles on the fly in Ambari. List activeProfiles = Arrays.asList(environment.getActiveProfiles()); - if (activeProfiles.contains(MetronRestConstants.DEV_PROFILE) || - activeProfiles.contains(MetronRestConstants.TEST_PROFILE)) { - auth.jdbcAuthentication().dataSource(dataSource) - .withUser("user").password("password").roles(SECURITY_ROLE_USER).and() - .withUser("user1").password("password").roles(SECURITY_ROLE_USER).and() - .withUser("user2").password("password").roles(SECURITY_ROLE_USER).and() - .withUser("admin").password("password").roles(SECURITY_ROLE_USER, SECURITY_ROLE_ADMIN); + if (activeProfiles.contains(MetronRestConstants.LDAP_PROFILE)) { + LOG.debug("Setting up LDAP authentication against {}.", providerUrl); + auth.ldapAuthentication() + .userDnPatterns(userDnPatterns) + .userSearchBase(userSearchBase) + .userSearchFilter(userSearchFilter) + .groupRoleAttribute(groupRoleAttribute) + .groupSearchFilter(groupSearchFilter) + .groupSearchBase(groupSearchBase) + .contextSource() + .url(providerUrl) + .managerDn(providerUserDn) + .managerPassword(providerPassword) + .and() + .passwordCompare() + .passwordEncoder(new LdapShaPasswordEncoder()) + .passwordAttribute(passwordAttribute); + } else if (activeProfiles.contains(MetronRestConstants.DEV_PROFILE) || + activeProfiles.contains(MetronRestConstants.TEST_PROFILE)) { + auth.jdbcAuthentication() + .dataSource(dataSource) + .withUser("user").password("password").roles(SECURITY_ROLE_USER).and() + .withUser("user1").password("password").roles(SECURITY_ROLE_USER).and() + .withUser("user2").password("password").roles(SECURITY_ROLE_USER).and() + .withUser("admin").password("password").roles(SECURITY_ROLE_USER, SECURITY_ROLE_ADMIN); } else { auth.jdbcAuthentication().dataSource(dataSource); } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java index b5f7765817..24c781c774 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java @@ -19,6 +19,11 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; +import java.util.List; +import java.util.stream.Collectors; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; @@ -35,4 +40,12 @@ public class UserController { public String user(Principal user) { return user.getName(); } + + @Secured("IS_AUTHENTICATED_FULLY") + @RequestMapping(path = "/whoami/roles", method = RequestMethod.GET) + public List user() { + UserDetails userDetails = (UserDetails) SecurityContextHolder.getContext(). + getAuthentication().getPrincipal(); + return userDetails.getAuthorities().stream().map(ga -> ga.getAuthority()).collect(Collectors.toList()); + } } diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml index 3eea24af3f..05ea571982 100644 --- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml +++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml @@ -52,9 +52,22 @@ storm: randomaccess.script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh batch.script.path: /usr/metron/${metron.version}/bin/start_hdfs_topology.sh - kerberos: enabled: false principal: metron@EXAMPLE.COM keytab: /etc/security/keytabs/metron.headless.keytab +ldap: + provider: + url: ldap://node1:33389 + userdn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + password: "admin-password" + user: + dn.patterns: uid={0},ou=people,dc=hadoop,dc=apache,dc=org + passwordAttribute: userPassword + searchBase: ou=people,dc=hadoop,dc=apache,dc=org + searchFilter: "" + group: + searchBase: ou=groups,dc=hadoop,dc=apache,dc=org + searchFilter: "member={0}" + roleAttribute: "cn" diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh index 7c89ae5073..b454b8e355 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh @@ -17,8 +17,8 @@ # limitations under the License. # -if [ -z "${METRON_JDBC_PASSWORD}" ]; then - echo "METRON_JDBC_PASSWORD unset. Exiting." +if [ -z "${METRON_JDBC_PASSWORD}" ] && [ -z "${METRON_LDAP_PASSWORD}" ]; then + echo "Authentication password unset. Exiting." exit 1 fi ## Join a list by a character @@ -35,6 +35,7 @@ METRON_REST_PORT=8082 METRON_SYSCONFIG="${METRON_SYSCONFIG:-/etc/default/metron}" METRON_LOG_DIR="${METRON_LOG_DIR:-/var/log/metron}" METRON_PID_FILE="${METRON_PID_FILE:-/var/run/metron/metron-rest.pid}" + PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib} INDEXING_CONTRIB=${INDEXING_CONTRIB:-$METRON_HOME/indexing_contrib} PARSER_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) @@ -112,6 +113,24 @@ METRON_REST_CLASSPATH+=":${indexing_files[0]}" echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}" +echo "METRON_JDBC_DRIVER=${METRON_JDBC_DRIVER}" +echo "METRON_JDBC_URL=${METRON_JDBC_URL}" +echo "METRON_JDBC_USERNAME=${METRON_JDBC_USERNAME}" +echo "METRON_JDBC_PLATFORM=${METRON_JDBC_PLATFORM}" + +echo "METRON_LDAP_URL=${METRON_LDAP_URL}" +echo "METRON_LDAP_USERDN=${METRON_LDAP_USERDN}" + +echo "METRON_LDAP_USER_PATTERN=${METRON_LDAP_USER_PATTERN}" +echo "METRON_LDAP_USER_PASSWORD=${METRON_LDAP_USER_PASSWORD}" +echo "METRON_LDAP_USER_SEARCHBASE=${METRON_LDAP_USER_SEARCHBASE}" +echo "METRON_LDAP_USER_SEARCHFILTER=${METRON_LDAP_USER_SEARCHFILTER}" + +echo "METRON_LDAP_GROUP_SEARCHBASE=${METRON_LDAP_GROUP_SEARCHBASE}" +echo "METRON_LDAP_GROUP_SEARCHFILTER=${METRON_LDAP_GROUP_SEARCHFILTER}" +echo "METRON_LDAP_GROUP_ROLE=${METRON_LDAP_GROUP_ROLE}" +echo "METRON_LDAP_SSL_TRUSTSTORE=${METRON_LDAP_SSL_TRUSTSTORE}" + #Use Solr daos if ra indexing writer set to Solr if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao" @@ -125,6 +144,11 @@ if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then METRON_SPRING_OPTIONS+=${METRON_WRITER_NAME} fi +if [ -n "${METRON_LDAP_SSL_TRUSTSTORE}" ]; then + METRON_JVMFLAGS+=" -Djavax.net.ssl.trustStore=${METRON_LDAP_SSL_TRUSTSTORE}" + METRON_JVMFLAGS+=" -Djavax.net.ssl.trustStorePassword=${METRON_LDAP_SSL_TRUSTSTORE_PASSWORD}" +fi + echo "Starting application" ${JAVA_HOME}/bin/java -Dhdp.version=${HDP_VERSION} ${METRON_JVMFLAGS} \ -cp ${METRON_REST_CLASSPATH} \