Skip to content

Commit

Permalink
refactor: optimize old attachment query parameters using index (#5363)
Browse files Browse the repository at this point in the history 
#### What type of PR is this?
/kind improvement
/area core
/milestone 2.13.x

#### What this PR does / why we need it:
使用索引功能优化附件列表查询

#### Does this PR introduce a user-facing change?
```release-note
使用索引功能优化附件列表查询
```
  • Loading branch information
@guqing
guqing committed Feb 20, 2024
1 parent 333422a commit 80e14e9
Show file tree
Hide file tree
Showing 11 changed files with 169 additions and 263 deletions.
27 changes: 27 additions & 0 deletions api/src/main/java/run/halo/app/extension/index/query/Not.java
View file
@@ -0,0 +1,27 @@
package run.halo.app.extension.index.query;

import static java.util.Objects.requireNonNull;

import java.util.Collections;
import java.util.NavigableSet;
import lombok.Getter;

@Getter
public class Not extends LogicalQuery {

private final Query negatedQuery;

public Not(Query negatedQuery) {
super(Collections.singleton(
requireNonNull(negatedQuery, "The negated query must not be null.")));
this.negatedQuery = negatedQuery;
}

@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var negatedResult = negatedQuery.matches(indexView);
var allIds = indexView.getAllIds();
allIds.removeAll(negatedResult);
return allIds;
}
}
4 changes: 4 additions & 0 deletions api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java
View file
Expand Up @@ -173,6 +173,10 @@ public static Query or(Query query1, Query query2, Collection<Query> additionalQ
return new Or(queries);
}

public static Query not(Query query) {
return new Not(query);
}

public static Query betweenLowerExclusive(String fieldName, String lowerValue,
String upperValue) {
return new Between(fieldName, lowerValue, false, upperValue, true);
Expand Down
10 changes: 10 additions & 0 deletions api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java
View file
Expand Up @@ -248,4 +248,14 @@ void containsTest() {
"104", "105"
);
}

@Test
void notTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet =
QueryFactory.not(QueryFactory.contains("firstName", "i")).matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "103", "104", "105"
);
}
}
167 changes: 48 additions & 119 deletions ...ion/src/main/java/run/halo/app/core/extension/attachment/endpoint/AttachmentEndpoint.java
View file
@@ -1,25 +1,27 @@
package run.halo.app.core.extension.attachment.endpoint;

import static io.swagger.v3.oas.annotations.media.Schema.RequiredMode.REQUIRED;
import static java.util.Comparator.comparing;
import static org.springdoc.core.fn.builders.apiresponse.Builder.responseBuilder;
import static org.springdoc.core.fn.builders.content.Builder.contentBuilder;
import static org.springdoc.core.fn.builders.schema.Builder.schemaBuilder;
import static org.springframework.boot.convert.ApplicationConversionService.getSharedInstance;
import static org.springframework.web.reactive.function.server.RequestPredicates.contentType;
import static run.halo.app.extension.ListResult.generateGenericClass;
import static run.halo.app.extension.index.query.QueryFactory.all;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.contains;
import static run.halo.app.extension.index.query.QueryFactory.in;
import static run.halo.app.extension.index.query.QueryFactory.isNull;
import static run.halo.app.extension.index.query.QueryFactory.not;
import static run.halo.app.extension.router.QueryParamBuildUtil.buildParametersFromType;
import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToPredicate;
import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToListOptions;

import io.swagger.v3.oas.annotations.media.ArraySchema;
import io.swagger.v3.oas.annotations.media.Schema;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Predicate;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.BooleanUtils;
import org.springdoc.core.fn.builders.requestbody.Builder;
import org.springdoc.webflux.core.fn.SpringdocRouteBuilder;
import org.springframework.data.domain.Sort;
Expand All @@ -42,11 +44,12 @@
import run.halo.app.core.extension.endpoint.CustomEndpoint;
import run.halo.app.core.extension.endpoint.SortResolver;
import run.halo.app.core.extension.service.AttachmentService;
import run.halo.app.extension.Comparators;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.extension.router.IListRequest;
import run.halo.app.extension.router.IListRequest.QueryListRequest;
import run.halo.app.extension.router.selector.LabelSelector;

@Slf4j
@Component
Expand Down Expand Up @@ -107,50 +110,35 @@ public RouterFunction<ServerResponse> endpoint() {

Mono<ServerResponse> search(ServerRequest request) {
var searchRequest = new SearchRequest(request);
return client.list(Group.class, group -> MetadataUtil.nullSafeLabels(group)
.containsKey(Group.HIDDEN_LABEL), null)
var groupListOptions = new ListOptions();
groupListOptions.setLabelSelector(LabelSelector.builder()
.exists(Group.HIDDEN_LABEL)
.build());
return client.listAll(Group.class, groupListOptions, Sort.unsorted())
.map(group -> group.getMetadata().getName())
.collectList()
.defaultIfEmpty(List.of())
.flatMap(groups -> client.list(Attachment.class,
searchRequest.toPredicate().and(visibleGroupPredicate(groups)),
searchRequest.toComparator(),
searchRequest.getPage(), searchRequest.getSize())
.flatMap(listResult -> ServerResponse.ok()
.contentType(MediaType.APPLICATION_JSON)
.bodyValue(listResult)
)
.flatMap(hiddenGroups -> client.listBy(Attachment.class,
searchRequest.toListOptions(hiddenGroups),
PageRequestImpl.of(searchRequest.getPage(), searchRequest.getSize(),
searchRequest.getSort())
)
.flatMap(listResult -> ServerResponse.ok()
.contentType(MediaType.APPLICATION_JSON)
.bodyValue(listResult)
)
);

}

static Predicate<Attachment> visibleGroupPredicate(List<String> hiddenGroups) {
return attachment -> {
if (!StringUtils.hasText(attachment.getSpec().getGroupName())) {
return true;
}
return !hiddenGroups.contains(attachment.getSpec().getGroupName());
};
}

public interface ISearchRequest extends IListRequest {

@Schema(description = "Display name of attachment")
Optional<String> getDisplayName();

@Schema(description = "Name of policy")
Optional<String> getPolicy();

@Schema(description = "Name of group")
Optional<String> getGroup();
@Schema(description = "Keyword for searching.")
Optional<String> getKeyword();

@Schema(description = "Filter attachments without group. This parameter will ignore group"
+ " parameter.")
Optional<Boolean> getUngrouped();

@Schema(description = "Name of user who uploaded the attachment")
Optional<String> getUploadedBy();

@ArraySchema(uniqueItems = true,
arraySchema = @Schema(name = "sort",
description = "Sort property and direction of the list result. Supported fields: "
Expand All @@ -159,7 +147,6 @@ public interface ISearchRequest extends IListRequest {
implementation = String.class,
example = "creationTimestamp,desc"))
Sort getSort();

}

public static class SearchRequest extends QueryListRequest implements ISearchRequest {
Expand All @@ -172,20 +159,8 @@ public SearchRequest(ServerRequest request) {
}

@Override
public Optional<String> getDisplayName() {
return Optional.ofNullable(queryParams.getFirst("displayName"))
.filter(StringUtils::hasText);
}

@Override
public Optional<String> getPolicy() {
return Optional.ofNullable(queryParams.getFirst("policy"))
.filter(StringUtils::hasText);
}

@Override
public Optional<String> getGroup() {
return Optional.ofNullable(queryParams.getFirst("group"))
public Optional<String> getKeyword() {
return Optional.ofNullable(queryParams.getFirst("keyword"))
.filter(StringUtils::hasText);
}

Expand All @@ -195,81 +170,35 @@ public Optional<Boolean> getUngrouped() {
.map(ungroupedStr -> getSharedInstance().convert(ungroupedStr, Boolean.class));
}

@Override
public Optional<String> getUploadedBy() {
return Optional.ofNullable(queryParams.getFirst("uploadedBy"))
.filter(StringUtils::hasText);
}

@Override
public Sort getSort() {
return SortResolver.defaultInstance.resolve(exchange);
var sort = SortResolver.defaultInstance.resolve(exchange);
sort = sort.and(Sort.by(
Sort.Order.desc("metadata.creationTimestamp"),
Sort.Order.asc("metadata.name")
));
return sort;
}

public Predicate<Attachment> toPredicate() {
Predicate<Attachment> displayNamePred = attachment -> getDisplayName()
.map(displayNameInParam -> {
String displayName = attachment.getSpec().getDisplayName();
return displayName.contains(displayNameInParam);
}).orElse(true);

Predicate<Attachment> policyPred = attachment -> getPolicy()
.map(policy -> Objects.equals(policy, attachment.getSpec().getPolicyName()))
.orElse(true);

Predicate<Attachment> groupPred = attachment -> getGroup()
.map(group -> Objects.equals(group, attachment.getSpec().getGroupName()))
.orElse(true);

Predicate<Attachment> ungroupedPred = attachment -> getUngrouped()
.filter(Boolean::booleanValue)
.map(ungrouped -> !StringUtils.hasText(attachment.getSpec().getGroupName()))
.orElseGet(() -> groupPred.test(attachment));
public ListOptions toListOptions(List<String> hiddenGroups) {
final var listOptions =
labelAndFieldSelectorToListOptions(getLabelSelector(), getFieldSelector());

Predicate<Attachment> uploadedByPred = attachment -> getUploadedBy()
.map(uploadedBy -> Objects.equals(uploadedBy, attachment.getSpec().getOwnerName()))
.orElse(true);


var selectorPred =
labelAndFieldSelectorToPredicate(getLabelSelector(), getFieldSelector());

return displayNamePred
.and(policyPred)
.and(ungroupedPred)
.and(uploadedByPred)
.and(selectorPred);
}
var fieldQuery = all();
if (getKeyword().isPresent()) {
fieldQuery = and(fieldQuery, contains("spec.displayName", getKeyword().get()));
}

public Comparator<Attachment> toComparator() {
var sort = getSort();
List<Comparator<Attachment>> comparators = new ArrayList<>();
var creationOrder = sort.getOrderFor("creationTimestamp");
if (creationOrder != null) {
Comparator<Attachment> comparator = comparing(
attachment -> attachment.getMetadata().getCreationTimestamp());
if (creationOrder.isDescending()) {
comparator = comparator.reversed();
}
comparators.add(comparator);
if (getUngrouped().isPresent() && BooleanUtils.isTrue(getUngrouped().get())) {
fieldQuery = and(fieldQuery, isNull("spec.groupName"));
}

var sizeOrder = sort.getOrderFor("size");
if (sizeOrder != null) {
Comparator<Attachment> comparator =
comparing(attachment -> attachment.getSpec().getSize());
if (sizeOrder.isDescending()) {
comparator = comparator.reversed();
}
comparators.add(comparator);
if (!hiddenGroups.isEmpty()) {
fieldQuery = and(fieldQuery, not(in("spec.groupName", hiddenGroups)));
}

// add default comparator
comparators.add(Comparators.compareCreationTimestamp(false));
comparators.add(Comparators.compareName(true));
return comparators.stream()
.reduce(Comparator::thenComparing)
.orElse(null);
listOptions.setFieldSelector(listOptions.getFieldSelector().andQuery(fieldQuery));
return listOptions;
}
}

Expand Down
34 changes: 33 additions & 1 deletion application/src/main/java/run/halo/app/infra/SchemeInitializer.java
View file
Expand Up @@ -6,6 +6,7 @@

import java.util.Set;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.boot.context.event.ApplicationContextInitializedEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.lang.NonNull;
Expand Down Expand Up @@ -191,7 +192,38 @@ public void onApplicationEvent(@NonNull ApplicationContextInitializedEvent event
// storage.halo.run
schemeManager.register(Group.class);
schemeManager.register(Policy.class);
schemeManager.register(Attachment.class);
schemeManager.register(Attachment.class, indexSpecs -> {
indexSpecs.add(new IndexSpec()
.setName("spec.displayName")
.setIndexFunc(simpleAttribute(Attachment.class,
attachment -> attachment.getSpec().getDisplayName()))
);
indexSpecs.add(new IndexSpec()
.setName("spec.policyName")
.setIndexFunc(simpleAttribute(Attachment.class,
attachment -> attachment.getSpec().getPolicyName()))
);
indexSpecs.add(new IndexSpec()
.setName("spec.groupName")
.setIndexFunc(simpleAttribute(Attachment.class, attachment -> {
var group = attachment.getSpec().getGroupName();
return StringUtils.isBlank(group) ? null : group;
}))
);
indexSpecs.add(new IndexSpec()
.setName("spec.ownerName")
.setIndexFunc(simpleAttribute(Attachment.class,
attachment -> attachment.getSpec().getOwnerName()))
);
indexSpecs.add(new IndexSpec()
.setName("spec.size")
.setIndexFunc(simpleAttribute(Attachment.class,
attachment -> {
var size = attachment.getSpec().getSize();
return size != null ? size.toString() : null;
}))
);
});
schemeManager.register(PolicyTemplate.class);
// metrics.halo.run
schemeManager.register(Counter.class);
Expand Down

0 comments on commit 80e14e9

Please sign in to comment.