diff --git a/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java b/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java index 29a709de..9f734410 100644 --- a/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java +++ b/dataset/src/main/java/org/keycloak/benchmark/dataset/DatasetResourceProvider.java @@ -38,8 +38,11 @@ import org.keycloak.models.AuthenticatedClientSessionModel; import org.keycloak.models.ClientModel; import org.keycloak.models.GroupModel; +import org.keycloak.models.GroupProvider; import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSessionFactory; +import org.keycloak.models.KeycloakSessionTask; +import org.keycloak.models.KeycloakSessionTaskWithResult; import org.keycloak.models.KeycloakUriInfo; import org.keycloak.models.PasswordPolicy; import org.keycloak.models.RealmModel; @@ -63,14 +66,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Optional; import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.IntStream; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS; +import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_GROUPS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_USERS; @@ -179,7 +182,7 @@ private void createRealmsImpl(Task task, DatasetConfig config, int startIndex, i // create each 100 groups per transaction as default case // (to avoid transaction timeouts when creating too many groups in one transaction) - createGroupsInMultipleTransactions(config, context, task); + createGroupsInMultipleTransactions(config, context, task, config.getGroupsPerRealm()); // Step 2 - create clients (Using single executor for now... For multiple executors run separate create-clients endpoint) for (int i = 0; i < config.getClientsPerRealm(); i += config.getClientsPerTransaction()) { @@ -212,27 +215,41 @@ private void createRealmsImpl(Task task, DatasetConfig config, int startIndex, i } } - private void createGroupsInMultipleTransactions(DatasetConfig config, RealmContext context, Task task) { - int groupsPerRealm = config.getGroupsPerRealm(); + private void createGroupsInMultipleTransactions(DatasetConfig config, RealmContext context, Task task, int topLevelCount) { boolean hierarchicalGroups = Boolean.parseBoolean(config.getGroupsWithHierarchy()); int hierarchyDepth = config.getGroupsHierarchyDepth(); - int countGroupsAtEachLevel = hierarchicalGroups ? config.getCountGroupsAtEachLevel() : groupsPerRealm; - int totalNumberOfGroups = hierarchicalGroups ? (int) Math.pow(countGroupsAtEachLevel, hierarchyDepth) : groupsPerRealm; - - for (int i = 0; i < totalNumberOfGroups; i += config.getGroupsPerTransaction()) { - int groupsStartIndex = i; - int groupEndIndex = hierarchicalGroups ? Math.min(groupsStartIndex + config.getGroupsPerTransaction(), totalNumberOfGroups) - : Math.min(groupsStartIndex + config.getGroupsPerTransaction(), config.getGroupsPerRealm()); + int countGroupsAtEachLevel = hierarchicalGroups ? config.getCountGroupsAtEachLevel() : 0; + String realmName = context.getRealm().getName(); + ExecutorHelper executor = new ExecutorHelper(config.getThreadsCount(), baseSession.getKeycloakSessionFactory(), config); + Long groupsCount = getGroupsCount(realmName); - logger.tracef("groupsStartIndex: %d, groupsEndIndex: %d", groupsStartIndex, groupEndIndex); + for (AtomicInteger index = new AtomicInteger(0); index.get() < topLevelCount; index.incrementAndGet()) { + KeycloakModelUtils.runJobInTransaction(baseSession.getKeycloakSessionFactory(), baseSession.getContext(), session -> { + RealmModel realm = session.realms().getRealmByName(realmName); + session.getContext().setRealm(realm); + String groupName = config.getGroupPrefix() + index.get(); - KeycloakModelUtils.runJobInTransactionWithTimeout(baseSession.getKeycloakSessionFactory(), - session -> createGroups(context, groupsStartIndex, groupEndIndex, hierarchicalGroups, hierarchyDepth, countGroupsAtEachLevel, session), - config.getTransactionTimeoutInSeconds()); + while (session.groups().getGroupByName(realm, null, groupName) != null) { + groupName = config.getGroupPrefix() + index.incrementAndGet(); + } - task.debug(logger, "Created %d groups in realm %s", context.getGroups().size(), context.getRealm().getName()); + String finalGroupName = groupName; + KeycloakModelUtils.runJobInTransaction(session.getKeycloakSessionFactory(), session.getContext(), s -> s.groups().createGroup(s.getContext().getRealm(), finalGroupName)); + logger.infof("Creating top-level group %s in realm %s", groupName, context.getRealm().getName()); + createGroupLevel(session, countGroupsAtEachLevel, hierarchyDepth, groupName, executor); + }); } - task.info(logger, "Created all %d groups in realm %s", context.getGroups().size(), context.getRealm().getName()); + + executor.waitForAllToFinish(); + + task.info(logger, "Created all %d groups in realm %s", getGroupsCount(realmName) - groupsCount, context.getRealm().getName()); + } + + private Long getGroupsCount(String realmName) { + return KeycloakModelUtils.runJobInTransactionWithResult(baseSession.getKeycloakSessionFactory(), baseSession.getContext(), session -> { + RealmModel realm = session.realms().getRealmByName(realmName); + return session.groups().getGroupsCount(realm, false); + }); } protected Response handleDatasetException(DatasetException de) { @@ -899,6 +916,65 @@ public Response takeDCUp() { return Response.ok(TaskResponse.statusMessage("Site " + siteName + " was marked as up.")).build(); } + @GET + @Path("/create-groups") + @NoCache + @Produces(MediaType.APPLICATION_JSON) + public Response createGroups() { + boolean started = false; + boolean taskAdded = false; + try { + DatasetConfig config = ConfigUtil.createConfigFromQueryParams(httpRequest, CREATE_GROUPS); + + Task task = Task.start("Creation of " + config.getCount() + " groups in the realm " + config.getRealmName()); + TaskManager taskManager = new TaskManager(baseSession); + Task existingTask = taskManager.addTaskIfNotInProgress(task, config.getTaskTimeout()); + if (existingTask != null) { + return Response.status(400).entity(TaskResponse.errorSomeTaskInProgress(existingTask, getStatusUrl())).build(); + } else { + taskAdded = true; + } + + logger.infof("Trigger creating groups with the configuration: %s", config); + + // Use the cache + RealmModel realm = baseSession.realms().getRealmByName(config.getRealmName()); + if (realm == null) { + throw new DatasetException("Realm '" + config.getRealmName() + "' not found"); + } + + int startIndex = ConfigUtil.findFreeEntityIndex(index -> { + String name = config.getGroupPrefix() + index; + return baseSession.groups().getGroupByName(realm, null, name) != null; + }); + config.setStart(startIndex); + + // Run this in separate thread to not block HTTP request + RealmContext context = new RealmContext(config); + + context.setRealm(realm); + + new Thread(() -> { + try { + createGroupsInMultipleTransactions(config, context, task, config.getCount()); + success(); + } catch (Exception e) { + KeycloakModelUtils.runJobInTransaction(baseSession.getKeycloakSessionFactory(), session + -> new TaskManager(session).removeExistingTask(false)); + } + }).start(); + started = true; + + return Response.ok(TaskResponse.taskStarted(task, getStatusUrl())).build(); + } catch (DatasetException de) { + return handleDatasetException(de); + } finally { + if (taskAdded && !started) { + new TaskManager(baseSession).removeExistingTask(false); + } + } + } + @Override public void close() { } @@ -1017,59 +1093,6 @@ private void createClients(RealmContext context, Task task, KeycloakSession sess task.debug(logger, "Created %d clients in realm %s", context.getClientCount(), context.getRealm().getName()); } - private String getGroupName(boolean hierarchicalGroups, int countGroupsAtEachLevel, String prefix, int currentCount) { - - if (!hierarchicalGroups) { - return prefix + currentCount; - } - if(currentCount == 0) { - return prefix + "0"; - } - // we are using "." separated paths in the group names, this is basically a number system with countGroupsAtEachLevel being the basis - // this allows us to find the parent group by trimming the group name even if the parent was created in previous transaction - StringBuilder groupName = new StringBuilder(); - if(countGroupsAtEachLevel == 1) { - // numbering system does not work for base 1 - groupName.append("0"); - IntStream.range(0, currentCount).forEach(i -> groupName.append(GROUP_NAME_SEPARATOR).append("0")); - return prefix + groupName; - } - - int leftover = currentCount; - while (leftover > 0) { - int digit = leftover % countGroupsAtEachLevel; - groupName.insert(0, digit + GROUP_NAME_SEPARATOR); - leftover = (leftover - digit) / countGroupsAtEachLevel; - } - return prefix + groupName.substring(0, groupName.length() - 1); - } - - private String getParentGroupName(String groupName) { - if (groupName == null || groupName.lastIndexOf(GROUP_NAME_SEPARATOR) < 0) { - return null; - } - return groupName.substring(0, groupName.lastIndexOf(GROUP_NAME_SEPARATOR)); - } - - private void createGroups(RealmContext context, int startIndex, int endIndex, boolean hierarchicalGroups, int hierarchyDepth, int countGroupsAtEachLevel, KeycloakSession session) { - RealmModel realm = context.getRealm(); - for (int i = startIndex; i < endIndex; i++) { - String groupName = getGroupName(hierarchicalGroups, countGroupsAtEachLevel, context.getConfig().getGroupPrefix(), i); - String parentGroupName = getParentGroupName(groupName); - - if (parentGroupName != null) { - Optional maybeParent = session.groups().searchForGroupByNameStream(realm, parentGroupName, true, -1, -1).findFirst(); - maybeParent.ifPresent(parent -> { - GroupModel groupModel = session.groups().createGroup(realm, groupName, parent); - context.groupCreated(groupModel); - }); - } else { - GroupModel groupModel = session.groups().createGroup(realm, groupName); - context.groupCreated(groupModel); - } - } - } - // Worker task to be triggered by single executor thread private void createUsers(RealmContext context, KeycloakSession session, int startIndex, int endIndex) { // Refresh the realm @@ -1231,4 +1254,30 @@ protected void success() { -> new TaskManager(session).removeExistingTask(true)); } + private void createGroupLevel(KeycloakSession session, int count, int depth, String parent, ExecutorHelper executor) { + RealmModel realm = session.getContext().getRealm(); + GroupProvider groups = session.groups(); + GroupModel parentGroup = groups.searchForGroupByNameStream(realm, parent, true, -1, -1).findAny().orElse(null); + + if (parentGroup == null) { + throw new RuntimeException("Parent group " + parent + " not found"); + } + + for (int index = 0; index < count; index++) { + String groupName = parent + GROUP_NAME_SEPARATOR + index; + + if (groups.getGroupByName(realm, parentGroup, groupName) != null) { + continue; + } + + logger.infof("Creating group %s", groupName); + KeycloakModelUtils.runJobInTransaction(session.getKeycloakSessionFactory(), session.getContext(), s -> s.groups().createGroup(realm, groupName, parentGroup)); + + executor.addTask(() -> { + for (int depthIndex = 0; depthIndex < depth; depthIndex++) { + KeycloakModelUtils.runJobInTransaction(session.getKeycloakSessionFactory(), session.getContext(), s -> createGroupLevel(s, count, depth - 1, groupName, executor)); + } + }); + } + } } diff --git a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java index d18e4a65..dffcbb38 100644 --- a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java +++ b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetConfig.java @@ -21,6 +21,7 @@ import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_AUTHZ_CLIENT; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_CLIENTS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_EVENTS; +import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_GROUPS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_OFFLINE_SESSIONS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_ORGS; import static org.keycloak.benchmark.dataset.config.DatasetOperation.CREATE_REALMS; @@ -57,14 +58,14 @@ public class DatasetConfig { private Integer lastToRemove; // Realm-name is required when creating many clients or users. The realm where clients/users will be created must already exists - @QueryParamFill(paramName = "realm-name", required = true, operations = { CREATE_CLIENTS, CREATE_USERS, LAST_CLIENT, LAST_USER, CREATE_AUTHZ_CLIENT }) + @QueryParamFill(paramName = "realm-name", required = true, operations = { CREATE_CLIENTS, CREATE_USERS, CREATE_GROUPS, LAST_CLIENT, LAST_USER, CREATE_AUTHZ_CLIENT }) private String realmName; // NOTE: Start index is not available as parameter as it will be "auto-detected" based on already created realms (clients, users) private Integer start; // Count of entities to be created. Entity is realm, client or user based on the operation - @QueryParamIntFill(paramName = "count", required = true, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) + @QueryParamIntFill(paramName = "count", required = true, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_EVENTS, CREATE_GROUPS, CREATE_OFFLINE_SESSIONS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) private Integer count; // Prefix for realm roles to create in every realm (in case of CREATE_REALMS) or to assign to users (in case of CREATE_USERS) @@ -107,27 +108,27 @@ public class DatasetConfig { private Integer clientRolesPerClient; // Prefix of groups to be created (in case of CREATE_REALMS operation) or assigned to the users (In case of CREATE_USERS and CREATE_REALMS operations) - @QueryParamFill(paramName = "group-prefix", defaultValue = "group-", operations = { CREATE_REALMS, CREATE_USERS }) + @QueryParamFill(paramName = "group-prefix", defaultValue = "group-", operations = { CREATE_REALMS, CREATE_USERS, CREATE_GROUPS }) private String groupPrefix; // Count of groups to be created in every created realm - @QueryParamIntFill(paramName = "groups-per-realm", defaultValue = 20, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-per-realm", defaultValue = 20, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer groupsPerRealm; // Number of groups to be created in one transaction - @QueryParamIntFill(paramName = "groups-per-transaction", defaultValue = 100, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-per-transaction", defaultValue = 100, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer groupsPerTransaction; // When this parameter is false only top level groups are created, groups and subgroups are created - @QueryParamFill(paramName = "groups-with-hierarchy", defaultValue = "false", operations = { CREATE_REALMS }) + @QueryParamFill(paramName = "groups-with-hierarchy", defaultValue = "false", operations = { CREATE_REALMS, CREATE_GROUPS }) private String groupsWithHierarchy; // Depth of the group hierarchy tree. Active if groups-with-hierarchy = true - @QueryParamIntFill(paramName = "groups-hierarchy-depth", defaultValue = 3, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-hierarchy-depth", defaultValue = 3, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer groupsHierarchyDepth; // Number of at each level of hierarchy. Each group will have this many subgroups. Active if groups-with-hierarchy = true - @QueryParamIntFill(paramName = "groups-count-each-level", defaultValue = 10, operations = { CREATE_REALMS }) + @QueryParamIntFill(paramName = "groups-count-each-level", defaultValue = 10, operations = { CREATE_REALMS, CREATE_GROUPS }) private Integer countGroupsAtEachLevel; @@ -165,7 +166,7 @@ public class DatasetConfig { // Transaction timeout used for transactions for creating objects @QueryParamIntFill(paramName = "transaction-timeout", defaultValue = 300, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, - CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) + CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS, CREATE_GROUPS }) private Integer transactionTimeoutInSeconds; // Count of users created in every transaction @@ -173,14 +174,14 @@ public class DatasetConfig { private Integer usersPerTransaction; // Count of worker threads concurrently creating entities - @QueryParamIntFill(paramName = "threads-count", operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, + @QueryParamIntFill(paramName = "threads-count", operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, CREATE_GROUPS, CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) private Integer threadsCount; // Timeout for the whole task. If timeout expires, then the existing task may not be terminated immediatelly. However it will be permitted to start another task // (EG. Send another HTTP request for creating realms), which can cause conflicts @QueryParamIntFill(paramName = "task-timeout", defaultValue = 3600, operations = { CREATE_REALMS, CREATE_CLIENTS, CREATE_USERS, - CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS }) + CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, REMOVE_REALMS, CREATE_AUTHZ_CLIENT, CREATE_ORGS, CREATE_GROUPS }) private Integer taskTimeout; // The client id of a client to which data is going to be provisioned diff --git a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java index e6b440e4..7f612832 100644 --- a/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java +++ b/dataset/src/main/java/org/keycloak/benchmark/dataset/config/DatasetOperation.java @@ -26,6 +26,7 @@ public enum DatasetOperation { CREATE_CLIENTS, CREATE_AUTHZ_CLIENT, CREATE_USERS, + CREATE_GROUPS, CREATE_EVENTS, CREATE_OFFLINE_SESSIONS, CREATE_ORGS, diff --git a/doc/dataset/modules/ROOT/pages/using-provider.adoc b/doc/dataset/modules/ROOT/pages/using-provider.adoc index 47ccec28..6a9a0bbc 100644 --- a/doc/dataset/modules/ROOT/pages/using-provider.adoc +++ b/doc/dataset/modules/ROOT/pages/using-provider.adoc @@ -107,9 +107,6 @@ The number of groups and the structure of the created groups can be managed by u `groups-per-realm`:: Total number of groups per realm. The default value is `20`. -`groups-per-transaction`:: Number of groups to be created in one transaction. -The default value is `100`. - `groups-with-hierarchy`:: `true` or `false`, the default value is `false`. With the default value, only top-level groups are created. With groups-with-hierarchy set to `true` a tree structure of groups is created; the depth of the tree is defined by the parameter `groups-hierarchy-depth` and `groups-count-each-level` defines how many subgroups each created group will have. @@ -131,6 +128,13 @@ The adopted subgroup naming convention uses a dot (`.`) in the group names which .../realms/master/dataset/create-realms?count=1&groups-with-hierarchy=true&groups-hierarchy-depth=3&groups-count-each-level=50 ---- +You can also create groups in an existing realm by invoking the `create-groups` endpoint and setting the `realm-name` parameter: + +.Example parameters +---- +.../realms/master/dataset/create-groups?realm-name=realm-0&count=10&groups-with-hierarchy=true&groups-hierarchy-depth=3&groups-count-each-level=5 +---- + === Create many events This is request to create 10M new events in the available realms with prefix `realm-`.