Add ability to read 'storeCustomAttributes' configuration file for id…

…entity providers from the UAA configuration file

[#144964151] https://www.pivotaltracker.com/story/show/144964151

model/src/main/java/org/cloudfoundry/identity/uaa/provider/ExternalIdentityProviderDefinition.java

@@ -34,6 +34,8 @@ public class ExternalIdentityProviderDefinition extends AbstractIdentityProvider
     public static final String USER_ATTRIBUTE_PREFIX = "user.attribute.";
     public static final String USER_NAME_ATTRIBUTE_NAME = "user_name";
 
+    public static final String STORE_CUSTOM_ATTRIBUTES_NAME = "storeCustomAttributes";
+
     public static final String EXTERNAL_GROUPS_WHITELIST = "externalGroupsWhitelist";
     public static final String ATTRIBUTE_MAPPINGS = "attributeMappings";
 

model/src/main/java/org/cloudfoundry/identity/uaa/provider/LdapIdentityProviderDefinition.java

@@ -50,6 +50,7 @@ public class LdapIdentityProviderDefinition extends ExternalIdentityProviderDefi
     public static final String LDAP_BASE_USER_DN_PATTERN = LDAP_PREFIX + "base.userDnPattern";
     public static final String LDAP_BASE_USER_DN_PATTERN_DELIMITER = LDAP_PREFIX + "base.userDnPatternDelimiter";
     public static final String LDAP_EMAIL_DOMAIN = LDAP_PREFIX + EMAIL_DOMAIN_ATTR;
+    public static final String LDAP_STORE_CUSTOM_ATTRIBUTES = LDAP_PREFIX + STORE_CUSTOM_ATTRIBUTES_NAME;
     public static final String LDAP_EXTERNAL_GROUPS_WHITELIST = LDAP_PREFIX + "externalGroupsWhitelist";
     public static final String LDAP_GROUP_FILE_GROUPS_AS_SCOPES = "ldap/ldap-groups-as-scopes.xml";
     public static final String LDAP_GROUP_FILE_GROUPS_MAP_TO_SCOPES = "ldap/ldap-groups-map-to-scopes.xml";

server/src/main/java/org/cloudfoundry/identity/uaa/provider/oauth/OauthIdentityProviderDefinitionFactoryBean.java

@@ -25,6 +25,7 @@
 import static org.cloudfoundry.identity.uaa.constants.OriginKeys.OAUTH20;
 import static org.cloudfoundry.identity.uaa.constants.OriginKeys.OIDC10;
 import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.ATTRIBUTE_MAPPINGS;
+import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.STORE_CUSTOM_ATTRIBUTES_NAME;
 import static org.springframework.util.StringUtils.hasText;
 
 public class OauthIdentityProviderDefinitionFactoryBean {
@@ -33,7 +34,7 @@ public class OauthIdentityProviderDefinitionFactoryBean {
     public OauthIdentityProviderDefinitionFactoryBean(Map<String, Map> definitions) {
         if (definitions != null) {
             for (String alias : definitions.keySet()) {
-                Map idpDefinitionMap = definitions.get(alias);
+                Map<String, Object> idpDefinitionMap = definitions.get(alias);
                 try {
                     String type = (String) idpDefinitionMap.get("type");
                     if(OAUTH20.equalsIgnoreCase(type)) {
@@ -58,13 +59,14 @@ else if(OIDC10.equalsIgnoreCase(type)) {
         }
     }
 
-    private void setCommonProperties(Map idpDefinitionMap, AbstractXOAuthIdentityProviderDefinition idpDefinition) {
+    protected void setCommonProperties(Map<String, Object> idpDefinitionMap, AbstractXOAuthIdentityProviderDefinition idpDefinition) {
         idpDefinition.setLinkText((String)idpDefinitionMap.get("linkText"));
         idpDefinition.setRelyingPartyId((String) idpDefinitionMap.get("relyingPartyId"));
         idpDefinition.setRelyingPartySecret((String) idpDefinitionMap.get("relyingPartySecret"));
         idpDefinition.setEmailDomain((List<String>) idpDefinitionMap.get("emailDomain"));
         idpDefinition.setShowLinkText(idpDefinitionMap.get("showLinkText") == null ? true : (boolean) idpDefinitionMap.get("showLinkText"));
         idpDefinition.setAddShadowUserOnLogin(idpDefinitionMap.get("addShadowUserOnLogin") == null ? true : (boolean) idpDefinitionMap.get("addShadowUserOnLogin"));
+        idpDefinition.setStoreCustomAttributes(idpDefinitionMap.get(STORE_CUSTOM_ATTRIBUTES_NAME) == null ? false : (boolean) idpDefinitionMap.get(STORE_CUSTOM_ATTRIBUTES_NAME));
         idpDefinition.setSkipSslValidation(idpDefinitionMap.get("skipSslValidation") == null ? false : (boolean) idpDefinitionMap.get("skipSslValidation"));
         idpDefinition.setTokenKey((String) idpDefinitionMap.get("tokenKey"));
         idpDefinition.setIssuer((String) idpDefinitionMap.get("issuer"));

server/src/main/java/org/cloudfoundry/identity/uaa/provider/saml/BootstrapSamlIdentityProviderConfigurator.java

@@ -34,6 +34,7 @@
 import static org.cloudfoundry.identity.uaa.provider.AbstractIdentityProviderDefinition.PROVIDER_DESCRIPTION;
 import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.ATTRIBUTE_MAPPINGS;
 import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.EXTERNAL_GROUPS_WHITELIST;
+import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.STORE_CUSTOM_ATTRIBUTES_NAME;
 import static org.springframework.util.StringUtils.hasText;
 
 public class BootstrapSamlIdentityProviderConfigurator implements InitializingBean {
@@ -121,6 +122,11 @@ public void setIdentityProviders(Map<String, Map<String, Object>> providers) {
             String providerDescription = (String)((Map)entry.getValue()).get(PROVIDER_DESCRIPTION);
             Boolean addShadowUserOnLogin = (Boolean)((Map)entry.getValue()).get("addShadowUserOnLogin");
             Boolean skipSslValidation = (Boolean)((Map)entry.getValue()).get("skipSslValidation");
+            Boolean storeCustomAttributes = (Boolean)((Map)entry.getValue()).get(STORE_CUSTOM_ATTRIBUTES_NAME);
+            if (storeCustomAttributes == null) {
+                storeCustomAttributes = false; //default value
+            }
+
             if (skipSslValidation==null) {
                 if (socketFactoryClassName != null) {
                     skipSslValidation = false;
@@ -133,6 +139,7 @@ public void setIdentityProviders(Map<String, Map<String, Object>> providers) {
             List<String> externalGroupsWhitelist = (List<String>) saml.get(EXTERNAL_GROUPS_WHITELIST);
             Map<String, Object> attributeMappings = (Map<String, Object>) saml.get(ATTRIBUTE_MAPPINGS);
             SamlIdentityProviderDefinition def = new SamlIdentityProviderDefinition();
+            def.setStoreCustomAttributes(storeCustomAttributes);
             if (hasText(providerDescription)) {
                 def.setProviderDescription(providerDescription);
             }

server/src/main/java/org/cloudfoundry/identity/uaa/util/LdapUtils.java

@@ -88,7 +88,9 @@ public static LdapIdentityProviderDefinition fromConfig(Map<String, Object> ldap
             return definition;
         }
 
-
+        if (ldapConfig.get(LdapIdentityProviderDefinition.LDAP_STORE_CUSTOM_ATTRIBUTES)!=null) {
+            definition.setStoreCustomAttributes((boolean) ldapConfig.get(LdapIdentityProviderDefinition.LDAP_STORE_CUSTOM_ATTRIBUTES));
+        }
 
         if (ldapConfig.get(LdapIdentityProviderDefinition.LDAP_EMAIL_DOMAIN)!=null) {
             definition.setEmailDomain((List<String>) ldapConfig.get(LdapIdentityProviderDefinition.LDAP_EMAIL_DOMAIN));

server/src/test/java/org/cloudfoundry/identity/uaa/config/IdentityProviderBootstrapTest.java

@@ -54,6 +54,7 @@
 import static org.cloudfoundry.identity.uaa.provider.AbstractIdentityProviderDefinition.PROVIDER_DESCRIPTION;
 import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.ATTRIBUTE_MAPPINGS;
 import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.EXTERNAL_GROUPS_WHITELIST;
+import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.STORE_CUSTOM_ATTRIBUTES_NAME;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -103,7 +104,9 @@ public void testLdapBootstrap() throws Exception {
         IdentityProviderProvisioning provisioning = new JdbcIdentityProviderProvisioning(jdbcTemplate);
         IdentityProviderBootstrap bootstrap = new IdentityProviderBootstrap(provisioning, new MockEnvironment());
         HashMap<String, Object> ldapConfig = new HashMap<>();
+
         ldapConfig.put(EMAIL_DOMAIN_ATTR, Arrays.asList("test.domain"));
+        ldapConfig.put(STORE_CUSTOM_ATTRIBUTES_NAME, true);
         final String idpDescription = "Test LDAP Provider Description";
         ldapConfig.put(PROVIDER_DESCRIPTION, idpDescription);
         List<String> attrMap = new ArrayList<>();
@@ -126,6 +129,7 @@ public void testLdapBootstrap() throws Exception {
         assertEquals(Arrays.asList("value"), ldapProvider.getConfig().getExternalGroupsWhitelist());
         assertEquals("first_name", ldapProvider.getConfig().getAttributeMappings().get("given_name"));
         assertEquals(idpDescription, ldapProvider.getConfig().getProviderDescription());
+        assertTrue(ldapProvider.getConfig().isStoreCustomAttributes());
     }
 
     @Test
@@ -264,11 +268,13 @@ public void testRemovedOAuthIdentityProviderIsInactive() throws Exception {
             assertNotNull(bootstrapOauthProvider.getLastModified());
             assertEquals(provider.getKey(), bootstrapOauthProvider.getType());
             assertTrue(bootstrapOauthProvider.isActive());
+            assertFalse(bootstrapOauthProvider.getConfig().isStoreCustomAttributes()); //default
             if (OIDC10.equals(provider.getKey())) {
                 assertEquals("code id_token", bootstrapOauthProvider.getConfig().getResponseType());
             } else {
                 assertEquals("code", bootstrapOauthProvider.getConfig().getResponseType());
             }
+
         }
 
         bootstrap.setOauthIdpDefinitions(null);

server/src/test/java/org/cloudfoundry/identity/uaa/provider/oauth/OauthIdentityProviderDefinitionFactoryBeanTest.java

@@ -0,0 +1,67 @@
+/*
+ * ****************************************************************************
+ *     Cloud Foundry
+ *     Copyright (c) [2009-2017] Pivotal Software, Inc. All Rights Reserved.
+ *
+ *     This product is licensed to you under the Apache License, Version 2.0 (the "License").
+ *     You may not use this product except in compliance with the License.
+ *
+ *     This product includes a number of subcomponents with
+ *     separate copyright notices and license terms. Your use of these
+ *     subcomponents is subject to the terms and conditions of the
+ *     subcomponent's license, as noted in the LICENSE file.
+ * ****************************************************************************
+ */
+
+package org.cloudfoundry.identity.uaa.provider.oauth;
+
+import org.cloudfoundry.identity.uaa.provider.OIDCIdentityProviderDefinition;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+import static org.cloudfoundry.identity.uaa.provider.ExternalIdentityProviderDefinition.STORE_CUSTOM_ATTRIBUTES_NAME;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class OauthIdentityProviderDefinitionFactoryBeanTest {
+
+    private OauthIdentityProviderDefinitionFactoryBean factoryBean;
+    private HashMap<String, Object> idpDefinitionMap;
+    private OIDCIdentityProviderDefinition providerDefinition;
+
+    @Before
+    public void setup() throws Exception {
+        factoryBean = new OauthIdentityProviderDefinitionFactoryBean(null);
+        providerDefinition = new OIDCIdentityProviderDefinition();
+        idpDefinitionMap = new HashMap<>();
+        idpDefinitionMap.put("authUrl", "http://auth.url");
+        idpDefinitionMap.put("relyingPartyId", "theClientId");
+        idpDefinitionMap.put("relyingPartySecret", "theClientSecret");
+        idpDefinitionMap.put("tokenKey", "key");
+        idpDefinitionMap.put("tokenUrl", "http://token.url");
+        idpDefinitionMap.put("tokenKeyUrl", "http://token-key.url");
+    }
+
+    @Test
+    public void store_custom_attributes_default() throws Exception {
+        factoryBean.setCommonProperties(idpDefinitionMap, providerDefinition);
+        assertFalse(providerDefinition.isStoreCustomAttributes());
+    }
+
+    @Test
+    public void store_custom_attributes_set_to_true() throws Exception {
+        idpDefinitionMap.put(STORE_CUSTOM_ATTRIBUTES_NAME, true);
+        factoryBean.setCommonProperties(idpDefinitionMap, providerDefinition);
+        assertTrue(providerDefinition.isStoreCustomAttributes());
+    }
+
+    @Test
+    public void store_custom_attributes_set_to_false() throws Exception {
+        idpDefinitionMap.put(STORE_CUSTOM_ATTRIBUTES_NAME, false);
+        factoryBean.setCommonProperties(idpDefinitionMap, providerDefinition);
+        assertFalse(providerDefinition.isStoreCustomAttributes());
+    }
+
+}

server/src/test/java/org/cloudfoundry/identity/uaa/provider/saml/BootstrapSamlIdentityProviderConfiguratorTests.java

@@ -100,6 +100,7 @@ public class BootstrapSamlIdentityProviderConfiguratorTests {
 
     public static String sampleYaml = "  providers:\n" +
         "    okta-local:\n" +
+        "      storeCustomAttributes: true\n" +
         "      idpMetadata: |\n" +
         "        " + testXmlFileData.replace("\n","\n        ") + "\n"+
         "      nameID: urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress\n" +
@@ -290,6 +291,7 @@ public void testGetIdentityProviders() throws Exception {
     public void testSetAddShadowUserOnLoginFromYaml() throws Exception {
         String yaml = "  providers:\n" +
             "    provider-without-shadow-user-definition:\n" +
+            "      storeCustomAttributes: true\n" +
             "      idpMetadata: |\n" +
             "        <?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
             "        <md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"provider1\">" +
@@ -300,6 +302,7 @@ public void testSetAddShadowUserOnLoginFromYaml() throws Exception {
             "        </md:EntityDescriptor>\n" +
             "      nameID: urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress\n" +
             "    provider-with-shadow-users-enabled:\n" +
+            "      storeCustomAttributes: false\n" +
             "      idpMetadata: |\n" +
             "        <?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
             "        <md:EntityDescriptor xmlns:md=\"urn:oasis:names:tc:SAML:2.0:metadata\" entityID=\"provider2\">" +
@@ -329,14 +332,17 @@ public void testSetAddShadowUserOnLoginFromYaml() throws Exception {
             switch (def.getIdpEntityAlias()) {
                 case "provider-without-shadow-user-definition" : {
                     assertTrue("If not specified, addShadowUserOnLogin is set to true", def.isAddShadowUserOnLogin());
+                    assertTrue("Override store custom attributes to true", def.isStoreCustomAttributes());
                     break;
                 }
                 case "provider-with-shadow-users-enabled" : {
                     assertTrue("addShadowUserOnLogin can be set to true", def.isAddShadowUserOnLogin());
+                    assertFalse("Default store custom attributes is false", def.isStoreCustomAttributes());
                     break;
                 }
                 case "provider-with-shadow-user-disabled" : {
                     assertFalse("addShadowUserOnLogin can be set to false", def.isAddShadowUserOnLogin());
+                    assertFalse("Default store custom attributes is false", def.isStoreCustomAttributes());
                     break;
                 }
                 default: fail(String.format("Unknown provider %s", def.getIdpEntityAlias()));

uaa/src/test/java/org/cloudfoundry/identity/uaa/login/BootstrapTests.java

@@ -501,6 +501,7 @@ public void all_properties_set() throws Exception {
         assertEquals(OIDC10, oidcProvider.getType());
         assertEquals(Collections.singletonList("requested_scope"), oidcProvider.getConfig().getScopes());
         assertEquals("code id_token", oidcProvider.getConfig().getResponseType());
+        assertTrue(oidcProvider.getConfig().isStoreCustomAttributes());
 
         IdentityProvider<AbstractXOAuthIdentityProviderDefinition> oauthProvider = idpProvisioning.retrieveByOrigin("my-oauth-provider", IdentityZone.getUaa().getId());
         assertNotNull(oauthProvider);
@@ -519,6 +520,7 @@ public void all_properties_set() throws Exception {
         assertEquals(Collections.singletonList("requested_scope"), oauthProvider.getConfig().getScopes());
         assertEquals(Collections.singletonList("example.com"), oauthProvider.getConfig().getEmailDomain());
         assertEquals("code", oauthProvider.getConfig().getResponseType());
+        assertTrue(oauthProvider.getConfig().isStoreCustomAttributes());
 
         IdentityZoneResolvingFilter filter = context.getBean(IdentityZoneResolvingFilter.class);
         assertThat(filter.getDefaultZoneHostnames(), containsInAnyOrder(uaa, login, "localhost", "host1.domain.com", "host2", "test3.localhost", "test4.localhost"));
@@ -629,6 +631,7 @@ public void all_properties_set() throws Exception {
         LdapIdentityProviderDefinition ldapConfig = ldapProvider.getConfig();
         assertFalse(ldapConfig.isAddShadowUserOnLogin());
         assertEquals("Test LDAP Provider Description", ldapConfig.getProviderDescription());
+        assertTrue(ldapConfig.isStoreCustomAttributes());
 
         //LDAP Group Validation
         assertEquals("ldap/ldap-groups-map-to-scopes.xml", ldapConfig.getLdapGroupFile());
@@ -643,6 +646,7 @@ public void all_properties_set() throws Exception {
         assertEquals("Test Okta Preview 1 Description", samlProvider.getConfig().getProviderDescription());
         assertEquals(SamlIdentityProviderDefinition.ExternalGroupMappingMode.EXPLICITLY_MAPPED, samlProvider.getConfig().getGroupMappingMode());
         assertTrue(samlProvider.getConfig().isSkipSslValidation());
+        assertTrue(samlProvider.getConfig().isStoreCustomAttributes());
 
         IdentityProvider<SamlIdentityProviderDefinition> samlProvider2 = providerProvisioning.retrieveByOrigin("okta-local-2", IdentityZone.getUaa().getId());
         assertEquals(SamlIdentityProviderDefinition.ExternalGroupMappingMode.AS_SCOPES, samlProvider2.getConfig().getGroupMappingMode());

uaa/src/test/resources/test/bootstrap/all-properties-set.yml

@@ -106,6 +106,7 @@ ldap:
   profile:
     file: ldap/ldap-search-and-bind.xml
   providerDescription: Test LDAP Provider Description
+  storeCustomAttributes: true
 links:
   global:
     passwd: "https://{zone.subdomain}.myaccountmanager.domain.com/z/{zone.id}/forgot_password"
@@ -150,10 +151,20 @@ login:
         tokenKeyUrl: null
         tokenUrl: http://my-token.com
         type: oauth2.0
+        storeCustomAttributes: true
       my-oidc-provider:
         attributeMappings:
           family_name: last_name
           given_name: first_name
+          user_name: <Attribute holding username in the OIDC ID Token>
+          external_groups:
+            - roles
+            - <other attribute holding roles or group memberships in the OIDC id_token>
+          user:
+            attribute:
+              name-of-attribute-in-uaa-id-token: name-of-attribute-in-provider-token
+              name-of-other-attribute-in-uaa-id-token: name-of-other-attribute-in-provider-token
+
         authUrl: http://my-auth.com
         linkText: My Oauth Provider
         relyingPartyId: uaa
@@ -167,6 +178,7 @@ login:
         tokenUrl: http://my-token.com
         type: oidc1.0
         userInfoUrl: http://my-token.com/userinfo
+        storeCustomAttributes: true
   prompt:
     password:
       text: Your Secret
@@ -288,6 +300,7 @@ login:
 
     providers:
       okta-local:
+        storeCustomAttributes: true
         iconUrl: http://link.to/icon.jpg
         idpMetadata: |
           <?xml version="1.0" encoding="UTF-8"?><!--