Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Group scalability upgrades #22700

Merged
merged 27 commits into from
Oct 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
2dfa4ca
Expand group storage and REST endpoints to allow for more flexibility…
alice-wondered Aug 17, 2023
3646815
update cache implementation for groups to allow for handling of searc…
alice-wondered Aug 21, 2023
9931174
add fine grained access information to populated hierarchy groups
alice-wondered Aug 21, 2023
324bd1f
add fine grained permission to non hierarchial returns on GroupsResou…
alice-wondered Aug 21, 2023
17134ed
Revert "update cache implementation for groups to allow for handling …
alice-wondered Aug 24, 2023
61f1a2e
updated ui to use new group api
edewit Aug 28, 2023
fa53c64
removed old workaround
edewit Aug 28, 2023
35d5fea
fixed type
edewit Aug 28, 2023
0efcc8b
Update model/infinispan/src/main/java/org/keycloak/models/cache/infin…
alice-wondered Aug 28, 2023
aa46456
Address comments on PR by adding default implementations to GroupProv…
alice-wondered Aug 28, 2023
8d9f21d
fixed move to test
edewit Aug 29, 2023
0306e50
Bypass cache on infinispan when searching or paginating subgroups. Re…
alice-wondered Aug 29, 2023
d427f3d
fixed single group fetch
edewit Aug 30, 2023
a8d69d2
Address issues with group search, including improper paging on deleti…
alice-wondered Oct 9, 2023
ac15515
Fix UI testing failing in multiple areas due to changes made to subgr…
alice-wondered Oct 16, 2023
7f5ca5e
Update member polling of subgroups to work recursively and drill down…
alice-wondered Oct 16, 2023
baf5f81
Modify concurrency test for creating groups to try and diagnose faili…
alice-wondered Oct 16, 2023
cd6475c
Attempt to get data up to three times if missing from group list to t…
alice-wondered Oct 16, 2023
cc8b1c2
Fix typescript formatting errors
alice-wondered Oct 17, 2023
6db7a70
Update keycloak admin client js library to support the new groups beh…
alice-wondered Oct 17, 2023
3cc41d8
Update the front end groups to use the new admin client additions rat…
alice-wondered Oct 18, 2023
af0b4a7
Update members subgroup loading to operate concurrently where possible
alice-wondered Oct 18, 2023
dfa5bef
Remove public modifier from function for brevity
alice-wondered Oct 18, 2023
16385fd
Deprecate methods that are no longer used on the RealmModel, update d…
alice-wondered Oct 23, 2023
fb97b64
Fix js linter error from upstream rebase
alice-wondered Oct 24, 2023
ba1d7c2
simplify storage changes
mhajas Oct 24, 2023
16e1ae9
fix tests
mhajas Oct 24, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ private void updatePolicy(Policy policy, String groupsClaim, Set<GroupPolicyRepr
config.put("groupsClaim", groupsClaim);
}

List<GroupModel> topLevelGroups = authorization.getRealm().getTopLevelGroupsStream().collect(Collectors.toList());
List<GroupModel> topLevelGroups = authorization.getKeycloakSession().groups().getTopLevelGroupsStream(authorization.getRealm()).collect(Collectors.toList());

for (GroupPolicyRepresentation.GroupDefinition definition : groups) {
GroupModel group = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,35 @@

package org.keycloak.representations.idm;

import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

/**
* @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
* @version $Revision: 1 $
*/
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public class GroupRepresentation {
// For an individual group these are the sufficient minimum fields
// to identify a group and operate on it in a basic way
protected String id;
protected String name;
protected String path;
protected String parentId;
protected Long subGroupCount;
// For navigating a hierarchy of groups, we can also include a minimum representation of subGroups
// These aren't populated by default and are only included as-needed
protected List<GroupRepresentation> subGroups;
protected Map<String, List<String>> attributes;
protected List<String> realmRoles;
protected Map<String, List<String>> clientRoles;
protected List<GroupRepresentation> subGroups;

private Map<String, Boolean> access;

public String getId() {
Expand All @@ -60,6 +72,22 @@ public void setPath(String path) {
this.path = path;
}

public String getParentId() {
return parentId;
}

public void setParentId(String parentId) {
this.parentId = parentId;
}

public Long getSubGroupCount() {
return subGroupCount;
}

public void setSubGroupCount(Long subGroupCount) {
this.subGroupCount = subGroupCount;
}

public List<String> getRealmRoles() {
return realmRoles;
}
Expand Down Expand Up @@ -92,6 +120,9 @@ public GroupRepresentation singleAttribute(String name, String value) {
}

public List<GroupRepresentation> getSubGroups() {
if(subGroups == null) {
subGroups = new ArrayList<>();
}
return subGroups;
}

Expand All @@ -106,4 +137,49 @@ public Map<String, Boolean> getAccess() {
public void setAccess(Map<String, Boolean> access) {
this.access = access;
}

public void merge(GroupRepresentation g) {
merge(this, g);
}

private void merge(GroupRepresentation g1, GroupRepresentation g2) {
if(g1.equals(g2)) {
Map<String, GroupRepresentation> g1Children = g1.getSubGroups().stream().collect(Collectors.toMap(GroupRepresentation::getId, g -> g));
Map<String, GroupRepresentation> g2Children = g2.getSubGroups().stream().collect(Collectors.toMap(GroupRepresentation::getId, g -> g));

g2Children.forEach((key, value) -> {
if (g1Children.containsKey(key)) {
merge(g1Children.get(key), value);
} else {
g1Children.put(key, value);
}
});
g1.setSubGroups(new ArrayList<>(g1Children.values()));
}
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
GroupRepresentation that = (GroupRepresentation) o;
boolean isEqual = Objects.equals(id, that.id) && Objects.equals(parentId, that.parentId);
if(isEqual) {
return true;
} else {
return Objects.equals(name, that.name) && Objects.equals(path, that.path);
}
}

@Override
public int hashCode() {
if(id == null) {
return Objects.hash(name, path);
}
return Objects.hash(id, parentId);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,31 @@ bin/kc.sh start --db postgres --db-username keycloak --db-url "jdbc:postgresql:/
The form action `RegistrationProfile` (displayed in the UI of authentication flows as `Profile Validation`) was removed from the codebase and also from all authentication flows. By default, it was in
the built-in registration flow of every realm. The validation of user attributes as well as creation of the user including all that user's attributes is handled by `RegistrationUserCreation` form action and
hence `RegistrationProfile` is not needed anymore. There is usually no further action needed in relation to this change, unless you used `RegistrationProfile` class in your own providers.

= Deprecated methods from data providers and models

* `RealmModel#getTopLevelGroupsStream()` and overloaded methods are now deprecated

= `GroupProvider` changes

A new method has been added to allow for searching and paging through top level groups.
If you implement this interface you will need to implement the following method:
[source,java]
----
Stream<GroupModel> getTopLevelGroupsStream(RealmModel realm,
String search,
Boolean exact,
Integer firstResult,
Integer maxResults)
----

= `GroupRepresentation` changes

* new field `subGroupCount` added to inform client how many subgroups are on any given group
* `subGroups` list is now only populated on queries that request hierarchy data
* This field is populated from the "bottom up" so can't be relied on for getting all subgroups for a group. Use a `GroupProvider` or request the subgroups from `GET {keycloak server}/realms/{realm}/groups/{group_id}/children`

= New endpoint for Group Admin API

Endpoint `GET {keycloak server}/realms/{realm}/groups/{group_id}/children` added as a way to get subgroups of specific groups that support pagination

Original file line number Diff line number Diff line change
Expand Up @@ -814,7 +814,8 @@ protected Stream<GroupModel> getKcSubGroups(RealmModel realm, GroupModel parentG
if (parentGroup == null) {
parentGroup = getKcGroupsPathGroup(realm);
}
return parentGroup == null ? realm.getTopLevelGroupsStream() : parentGroup.getSubGroupsStream();
return parentGroup == null ? session.groups().getTopLevelGroupsStream(realm) :
parentGroup.getSubGroupsStream();
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,18 @@ public interface GroupResource {
@DELETE
void remove();

/**
* Get the paginated list of subgroups belonging to this group
*
* @param first
* @param max
* @param full
*/
@GET
@Path("children")
@Produces(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
@Consumes(jakarta.ws.rs.core.MediaType.APPLICATION_JSON)
List<GroupRepresentation> getSubGroups(@QueryParam("first") Integer first, @QueryParam("max") Integer max, @QueryParam("briefRepresentation") Boolean briefRepresentation);

/**
* Set or create child. This will just set the parent if it exists. Create it and set the parent
Expand Down
8 changes: 4 additions & 4 deletions js/apps/admin-ui/cypress/e2e/sessions_test.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ import SidebarPage from "../support/pages/admin-ui/SidebarPage";
import SessionsPage from "../support/pages/admin-ui/manage/sessions/SessionsPage";
import CommonPage from "../support/pages/CommonPage";
import ListingPage from "../support/pages/admin-ui/ListingPage";
import GroupPage from "../support/pages/admin-ui/manage/groups/GroupPage";
import { keycloakBefore } from "../support/util/keycloak_hooks";
import PageObject from "../support/pages/admin-ui/components/PageObject";

const loginPage = new LoginPage();
const sidebarPage = new SidebarPage();
const sessionsPage = new SessionsPage();
const commonPage = new CommonPage();
const listingPage = new ListingPage();
const groupPage = new GroupPage();
const page = new PageObject();

describe("Sessions test", () => {
const admin = "admin";
Expand Down Expand Up @@ -42,12 +42,12 @@ describe("Sessions test", () => {
it("search existing session", () => {
listingPage.searchItem(admin, false);
listingPage.itemExist(admin, true);
groupPage.assertNoSearchResultsMessageExist(false);
page.assertEmptyStateExist(false);
});

it("search non-existant session", () => {
listingPage.searchItem("non-existant-session", false);
groupPage.assertNoSearchResultsMessageExist(true);
page.assertEmptyStateExist(true);
});
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ export default class PageObject {
return this;
}

protected assertEmptyStateExist(exist: boolean) {
assertEmptyStateExist(exist: boolean) {
if (exist) {
cy.get(this.#emptyStateDiv).should("exist").should("be.visible");
} else {
Expand Down
Loading
Loading