Skip to content

Commit

Permalink
refactor: optimize user query using index
Browse files Browse the repository at this point in the history
  • Loading branch information
@guqing
guqing committed Feb 22, 2024
1 parent 9e67671 commit 5e3bde3
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 92 deletions.
2 changes: 2 additions & 0 deletions api/src/main/java/run/halo/app/core/extension/User.java
View file
Expand Up @@ -33,6 +33,8 @@ public class User extends AbstractExtension {
public static final String VERSION = "v1alpha1";
public static final String KIND = "User";

public static final String USER_RELATED_ROLES_INDEX = "roles";

public static final String ROLE_NAMES_ANNO = "rbac.authorization.halo.run/role-names";

public static final String EMAIL_TO_VERIFY = "halo.run/email-to-verify";
Expand Down
27 changes: 13 additions & 14 deletions application/src/main/java/run/halo/app/core/extension/endpoint/StatsEndpoint.java
View file
@@ -1,6 +1,5 @@
package run.halo.app.core.extension.endpoint;

import static java.lang.Boolean.parseBoolean;
import static org.springdoc.core.fn.builders.apiresponse.Builder.responseBuilder;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.equal;
Expand All @@ -17,10 +16,10 @@
import run.halo.app.core.extension.User;
import run.halo.app.core.extension.content.Post;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;

/**
* Stats endpoint.
Expand Down Expand Up @@ -61,18 +60,18 @@ Mono<ServerResponse> getStats(ServerRequest request) {
stats.setUpvotes(stats.getUpvotes() + counter.getUpvote());
return stats;
})
.flatMap(stats -> client.list(User.class,
user -> {
var labels = MetadataUtil.nullSafeLabels(user);
return user.getMetadata().getDeletionTimestamp() == null
&& !parseBoolean(labels.getOrDefault(User.HIDDEN_USER_LABEL, "false"));
},
null)
.count()
.map(count -> {
stats.setUsers(count.intValue());
return stats;
}))
.flatMap(stats -> {
var listOptions = new ListOptions();
listOptions.setLabelSelector(LabelSelector.builder()
.notEq(User.HIDDEN_USER_LABEL, "true")
.build()
);
listOptions.setFieldSelector(
FieldSelector.of(isNull("metadata.deletionTimestamp")));
return client.listBy(User.class, listOptions, PageRequestImpl.ofSize(1))
.doOnNext(result -> stats.setUsers((int) result.getTotal()))
.thenReturn(stats);
})
.flatMap(stats -> {
var listOptions = new ListOptions();
listOptions.setFieldSelector(FieldSelector.of(
Expand Down
108 changes: 41 additions & 67 deletions application/src/main/java/run/halo/app/core/extension/endpoint/UserEndpoint.java
View file
@@ -1,7 +1,6 @@
package run.halo.app.core.extension.endpoint;

import static io.swagger.v3.oas.annotations.media.Schema.RequiredMode.REQUIRED;
import static java.util.Comparator.comparing;
import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
import static org.apache.commons.lang3.StringUtils.defaultIfBlank;
import static org.springdoc.core.fn.builders.apiresponse.Builder.responseBuilder;
Expand All @@ -11,8 +10,12 @@
import static org.springdoc.core.fn.builders.schema.Builder.schemaBuilder;
import static org.springframework.web.reactive.function.server.RequestPredicates.contentType;
import static run.halo.app.extension.ListResult.generateGenericClass;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.contains;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.or;
import static run.halo.app.extension.router.QueryParamBuildUtil.buildParametersFromType;
import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToPredicate;
import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToListOptions;
import static run.halo.app.security.authorization.AuthorityUtils.authoritiesToRoles;

import com.fasterxml.jackson.core.type.TypeReference;
Expand All @@ -25,9 +28,7 @@
import io.swagger.v3.oas.annotations.media.Schema;
import java.security.Principal;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
Expand All @@ -37,7 +38,6 @@
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.RequiredArgsConstructor;
Expand Down Expand Up @@ -76,12 +76,14 @@
import run.halo.app.core.extension.service.EmailVerificationService;
import run.halo.app.core.extension.service.RoleService;
import run.halo.app.core.extension.service.UserService;
import run.halo.app.extension.Comparators;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ListResult;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.extension.router.IListRequest;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.infra.AnonymousUserConst;
import run.halo.app.infra.SystemConfigurableEnvironmentFetcher;
import run.halo.app.infra.SystemSetting;
Expand Down Expand Up @@ -676,7 +678,7 @@ public static class UserPermission {

}

public class ListRequest extends IListRequest.QueryListRequest {
public static class ListRequest extends IListRequest.QueryListRequest {

private final ServerWebExchange exchange;

Expand All @@ -703,63 +705,38 @@ public String getRole() {
implementation = String.class,
example = "creationTimestamp,desc"))
public Sort getSort() {
return SortResolver.defaultInstance.resolve(exchange);
var sort = SortResolver.defaultInstance.resolve(exchange);
sort = sort.and(Sort.by("metadata.creationTimestamp", "metadata.name").descending());
return sort;
}

/**
* Converts query parameters to user predicate.
*
* @return user predicate to filter users
* Converts query parameters to list options.
*/
public Predicate<User> toPredicate() {
Predicate<User> keywordPredicate = user -> {
var keyword = getKeyword();
if (StringUtils.isBlank(keyword)) {
return true;
}
var username = user.getMetadata().getName();
var displayName = user.getSpec().getDisplayName();
return StringUtils.containsIgnoreCase(displayName, keyword)
|| keyword.equalsIgnoreCase(username);
};

Predicate<User> rolePredicate = user -> {
var roleName = getRole();
if (StringUtils.isBlank(roleName)) {
return true;
}
var roleNamesAnno = MetadataUtil.nullSafeAnnotations(user)
.get(User.ROLE_NAMES_ANNO);
if (StringUtils.isBlank(roleNamesAnno)) {
return false;
}
Set<String> roleNames = JsonUtils.jsonToObject(roleNamesAnno,
new TypeReference<>() {
});
return roleNames.contains(roleName);
};
return keywordPredicate
.and(rolePredicate)
.and(labelAndFieldSelectorToPredicate(getLabelSelector(), getFieldSelector()));
}
public ListOptions toListOptions() {
var listOptions =
labelAndFieldSelectorToListOptions(getLabelSelector(), getFieldSelector());

var fieldQuery = listOptions.getFieldSelector().query();
if (StringUtils.isNotBlank(getKeyword())) {
fieldQuery = and(
fieldQuery,
or(
contains("spec.displayName", getKeyword()),
equal("metadata.name", getKeyword())
)
);
}

public Comparator<User> toComparator() {
var sort = getSort();
var ctOrder = sort.getOrderFor("creationTimestamp");
List<Comparator<User>> comparators = new ArrayList<>();
if (ctOrder != null) {
Comparator<User> comparator =
comparing(user -> user.getMetadata().getCreationTimestamp());
if (ctOrder.isDescending()) {
comparator = comparator.reversed();
}
comparators.add(comparator);
if (StringUtils.isNotBlank(getRole())) {
fieldQuery = and(
fieldQuery,
equal(User.USER_RELATED_ROLES_INDEX, getRole())
);
}
comparators.add(Comparators.compareCreationTimestamp(false));
comparators.add(Comparators.compareName(true));
return comparators.stream()
.reduce(Comparator::thenComparing)
.orElse(null);

listOptions.setFieldSelector(FieldSelector.of(fieldQuery));
return listOptions;
}
}

Expand All @@ -770,15 +747,12 @@ record ListedUser(@Schema(requiredMode = REQUIRED) User user,
Mono<ServerResponse> list(ServerRequest request) {
return Mono.just(request)
.map(UserEndpoint.ListRequest::new)
.flatMap(listRequest -> {
var predicate = listRequest.toPredicate();
var comparator = listRequest.toComparator();
return client.list(User.class,
predicate,
comparator,
listRequest.getPage(),
listRequest.getSize());
})
.flatMap(listRequest -> client.listBy(User.class, listRequest.toListOptions(),
PageRequestImpl.of(
listRequest.getPage(), listRequest.getSize(),
listRequest.getSort()
)
))
.flatMap(this::toListedUser)
.flatMap(listResult -> ServerResponse.ok().bodyValue(listResult));
}
Expand Down
23 changes: 13 additions & 10 deletions application/src/main/java/run/halo/app/infra/DefaultInitializationStateGetter.java
View file
@@ -1,15 +1,19 @@
package run.halo.app.infra;

import static org.apache.commons.lang3.BooleanUtils.isTrue;
import static run.halo.app.extension.index.query.QueryFactory.isNull;

import java.util.concurrent.atomic.AtomicBoolean;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Mono;
import run.halo.app.core.extension.User;
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;

/**
* <p>A cache that caches system setup state.</p>
Expand Down Expand Up @@ -50,16 +54,15 @@ public Mono<Boolean> dataInitialized() {
}

private Mono<Boolean> hasUser() {
return client.list(User.class,
user -> {
var labels = MetadataUtil.nullSafeLabels(user);
return isNotTrue(labels.get("halo.run/hidden-user"));
}, null, 1, 10)
var listOptions = new ListOptions();
listOptions.setLabelSelector(LabelSelector.builder()
.notEq(User.HIDDEN_USER_LABEL, "true")
.build()
);
listOptions.setFieldSelector(
FieldSelector.of(isNull("metadata.deletionTimestamp")));
return client.listBy(User.class, listOptions, PageRequestImpl.ofSize(1))
.map(result -> result.getTotal() > 0)
.defaultIfEmpty(false);
}

static boolean isNotTrue(String value) {
return !Boolean.parseBoolean(value);
}
}
22 changes: 21 additions & 1 deletion application/src/main/java/run/halo/app/infra/SchemeInitializer.java
View file
Expand Up @@ -4,6 +4,7 @@
import static run.halo.app.extension.index.IndexAttributeFactory.multiValueAttribute;
import static run.halo.app.extension.index.IndexAttributeFactory.simpleAttribute;

import com.fasterxml.jackson.core.type.TypeReference;
import java.util.Set;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -44,9 +45,11 @@
import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.DefaultSchemeManager;
import run.halo.app.extension.DefaultSchemeWatcherManager;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.extension.Secret;
import run.halo.app.extension.index.IndexSpec;
import run.halo.app.extension.index.IndexSpecRegistryImpl;
import run.halo.app.infra.utils.JsonUtils;
import run.halo.app.migration.Backup;
import run.halo.app.plugin.extensionpoint.ExtensionDefinition;
import run.halo.app.plugin.extensionpoint.ExtensionPointDefinition;
Expand All @@ -69,7 +72,24 @@ public void onApplicationEvent(@NonNull ApplicationContextInitializedEvent event
schemeManager.register(ExtensionDefinition.class);

schemeManager.register(RoleBinding.class);
schemeManager.register(User.class);
schemeManager.register(User.class, indexSpecs -> {
indexSpecs.add(new IndexSpec()
.setName("spec.displayName")
.setIndexFunc(
simpleAttribute(User.class, user -> user.getSpec().getDisplayName())));
indexSpecs.add(new IndexSpec()
.setName(User.USER_RELATED_ROLES_INDEX)
.setIndexFunc(multiValueAttribute(User.class, user -> {
var roleNamesAnno = MetadataUtil.nullSafeAnnotations(user)
.get(User.ROLE_NAMES_ANNO);
if (StringUtils.isBlank(roleNamesAnno)) {
return Set.of();
}
return JsonUtils.jsonToObject(roleNamesAnno,
new TypeReference<>() {
});
})));
});
schemeManager.register(ReverseProxy.class);
schemeManager.register(Setting.class);
schemeManager.register(AnnotationSetting.class);
Expand Down

0 comments on commit 5e3bde3

Please sign in to comment.