Group scalability upgrades (#22700)

closes #22372 


Co-authored-by: Erik Jan de Wit <erikjan.dewit@gmail.com>
Co-authored-by: Pedro Igor <pigor.craveiro@gmail.com>
Co-authored-by: Michal Hajas <mhajas@redhat.com>

authz/policy/common/src/main/java/org/keycloak/authorization/policy/provider/group/GroupPolicyProviderFactory.java

@@ -164,7 +164,7 @@ private void updatePolicy(Policy policy, String groupsClaim, Set<GroupPolicyRepr
             config.put("groupsClaim", groupsClaim);
         }
 
-        List<GroupModel> topLevelGroups = authorization.getRealm().getTopLevelGroupsStream().collect(Collectors.toList());
+        List<GroupModel> topLevelGroups = authorization.getKeycloakSession().groups().getTopLevelGroupsStream(authorization.getRealm()).collect(Collectors.toList());
 
         for (GroupPolicyRepresentation.GroupDefinition definition : groups) {
             GroupModel group = null;

core/src/main/java/org/keycloak/representations/idm/GroupRepresentation.java

@@ -17,23 +17,35 @@
 
 package org.keycloak.representations.idm;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
  * @version $Revision: 1 $
  */
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
 public class GroupRepresentation {
+    // For an individual group these are the sufficient minimum fields
+    // to identify a group and operate on it in a basic way
     protected String id;
     protected String name;
     protected String path;
+    protected String parentId;
+    protected Long subGroupCount;
+    // For navigating a hierarchy of groups, we can also include a minimum representation of subGroups
+    // These aren't populated by default and are only included as-needed
+    protected List<GroupRepresentation> subGroups;
     protected Map<String, List<String>>  attributes;
     protected List<String> realmRoles;
     protected Map<String, List<String>> clientRoles;
-    protected List<GroupRepresentation> subGroups;
+
     private Map<String, Boolean> access;
 
     public String getId() {
@@ -60,6 +72,22 @@ public void setPath(String path) {
         this.path = path;
     }
 
+    public String getParentId() {
+        return parentId;
+    }
+
+    public void setParentId(String parentId) {
+        this.parentId = parentId;
+    }
+
+    public Long getSubGroupCount() {
+        return subGroupCount;
+    }
+
+    public void setSubGroupCount(Long subGroupCount) {
+        this.subGroupCount = subGroupCount;
+    }
+
     public List<String> getRealmRoles() {
         return realmRoles;
     }
@@ -92,6 +120,9 @@ public GroupRepresentation singleAttribute(String name, String value) {
     }
 
     public List<GroupRepresentation> getSubGroups() {
+        if(subGroups == null) {
+            subGroups = new ArrayList<>();
+        }
         return subGroups;
     }
 
@@ -106,4 +137,49 @@ public Map<String, Boolean> getAccess() {
     public void setAccess(Map<String, Boolean> access) {
         this.access = access;
     }
+
+    public void merge(GroupRepresentation g) {
+        merge(this, g);
+    }
+
+    private void merge(GroupRepresentation g1, GroupRepresentation g2) {
+        if(g1.equals(g2)) {
+            Map<String, GroupRepresentation> g1Children = g1.getSubGroups().stream().collect(Collectors.toMap(GroupRepresentation::getId, g -> g));
+            Map<String, GroupRepresentation> g2Children = g2.getSubGroups().stream().collect(Collectors.toMap(GroupRepresentation::getId, g -> g));
+
+            g2Children.forEach((key, value) -> {
+                if (g1Children.containsKey(key)) {
+                    merge(g1Children.get(key), value);
+                } else {
+                    g1Children.put(key, value);
+                }
+            });
+            g1.setSubGroups(new ArrayList<>(g1Children.values()));
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GroupRepresentation that = (GroupRepresentation) o;
+        boolean isEqual = Objects.equals(id, that.id) && Objects.equals(parentId, that.parentId);
+        if(isEqual) {
+            return true;
+        } else {
+            return Objects.equals(name, that.name) && Objects.equals(path, that.path);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        if(id == null) {
+            return Objects.hash(name, path);
+        }
+        return Objects.hash(id, parentId);
+    }
 }

docs/documentation/upgrading/topics/keycloak/changes-23_0_0.adoc

@@ -107,3 +107,31 @@ bin/kc.sh start --db postgres --db-username keycloak --db-url "jdbc:postgresql:/
 The form action `RegistrationProfile` (displayed in the UI of authentication flows as `Profile Validation`) was removed from the codebase and also from all authentication flows. By default, it was in
 the built-in registration flow of every realm. The validation of user attributes as well as creation of the user including all that user's attributes is handled by `RegistrationUserCreation` form action and
 hence `RegistrationProfile` is not needed anymore. There is usually no further action needed in relation to this change, unless you used `RegistrationProfile` class in your own providers.
+
+= Deprecated methods from data providers and models
+
+* `RealmModel#getTopLevelGroupsStream()` and overloaded methods are now deprecated
+
+= `GroupProvider` changes
+
+A new method has been added to allow for searching and paging through top level groups.
+If you implement this interface you will need to implement the following method:
+[source,java]
+----
+Stream<GroupModel> getTopLevelGroupsStream(RealmModel realm,
+                                           String search,
+                                           Boolean exact,
+                                           Integer firstResult,
+                                           Integer maxResults)
+----
+
+= `GroupRepresentation` changes
+
+* new field `subGroupCount` added to inform client how many subgroups are on any given group
+* `subGroups` list is now only populated on queries that request hierarchy data
+  * This field is populated from the "bottom up" so can't be relied on for getting all subgroups for a group. Use a `GroupProvider` or request the subgroups from `GET {keycloak server}/realms/{realm}/groups/{group_id}/children`
+
+= New endpoint for Group Admin API
+
+Endpoint `GET {keycloak server}/realms/{realm}/groups/{group_id}/children` added as a way to get subgroups of specific groups that support pagination
+

federation/ldap/src/main/java/org/keycloak/storage/ldap/mappers/membership/group/GroupLDAPStorageMapper.java

@@ -814,7 +814,8 @@ protected Stream<GroupModel> getKcSubGroups(RealmModel realm, GroupModel parentG
         if (parentGroup == null) {
             parentGroup = getKcGroupsPathGroup(realm);
         }
-        return parentGroup == null ? realm.getTopLevelGroupsStream() : parentGroup.getSubGroupsStream();
+        return parentGroup == null ? session.groups().getTopLevelGroupsStream(realm) :
+            parentGroup.getSubGroupsStream();
     }
 
     /**

integration/admin-client-jee/src/main/java/org/keycloak/admin/client/resource/GroupResource.java

@@ -85,6 +85,18 @@ public interface GroupResource {
     @DELETE
     void remove();
 
+    /**
+     * Get the paginated list of subgroups belonging to this group
+     *
+     * @param first
+     * @param max
+     * @param full
+     */
+    @GET
+    @Path("children")
+    @Produces(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
+    @Consumes(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
+    List<GroupRepresentation> getSubGroups(@QueryParam("first") Integer first, @QueryParam("max") Integer max, @QueryParam("briefRepresentation") Boolean briefRepresentation);
 
     /**
      * Set or create child.  This will just set the parent if it exists.  Create it and set the parent

js/apps/admin-ui/cypress/e2e/sessions_test.spec.ts

@@ -3,15 +3,15 @@ import SidebarPage from "../support/pages/admin-ui/SidebarPage";
 import SessionsPage from "../support/pages/admin-ui/manage/sessions/SessionsPage";
 import CommonPage from "../support/pages/CommonPage";
 import ListingPage from "../support/pages/admin-ui/ListingPage";
-import GroupPage from "../support/pages/admin-ui/manage/groups/GroupPage";
 import { keycloakBefore } from "../support/util/keycloak_hooks";
+import PageObject from "../support/pages/admin-ui/components/PageObject";
 
 const loginPage = new LoginPage();
 const sidebarPage = new SidebarPage();
 const sessionsPage = new SessionsPage();
 const commonPage = new CommonPage();
 const listingPage = new ListingPage();
-const groupPage = new GroupPage();
+const page = new PageObject();
 
 describe("Sessions test", () => {
   const admin = "admin";
@@ -42,12 +42,12 @@ describe("Sessions test", () => {
     it("search existing session", () => {
       listingPage.searchItem(admin, false);
       listingPage.itemExist(admin, true);
-      groupPage.assertNoSearchResultsMessageExist(false);
+      page.assertEmptyStateExist(false);
     });
 
     it("search non-existant session", () => {
       listingPage.searchItem("non-existant-session", false);
-      groupPage.assertNoSearchResultsMessageExist(true);
+      page.assertEmptyStateExist(true);
     });
   });
 

js/apps/admin-ui/cypress/support/pages/admin-ui/components/PageObject.ts

@@ -329,7 +329,7 @@ export default class PageObject {
     return this;
   }
 
-  protected assertEmptyStateExist(exist: boolean) {
+  assertEmptyStateExist(exist: boolean) {
     if (exist) {
       cy.get(this.#emptyStateDiv).should("exist").should("be.visible");
     } else {