Skip to content
Permalink
Browse files
sort results of search index by matched count (#653)
implement: #630

Change-Id: I6cab6741f2e0dfcdb731ffc099334508e33ca2e1
  • Loading branch information
javeme authored and zhoney committed Sep 10, 2019
1 parent 94c2d34 commit d47f309da13d5bb38d256e193e6a5a9142299e62
Showing 33 changed files with 649 additions and 186 deletions.
@@ -75,6 +75,11 @@ public boolean supportsQueryByPage() {
return true;
}

@Override
public boolean supportsQuerySortByInputIds() {
return false;
}

@Override
public boolean supportsDeleteEdgeByLabel() {
return true;
@@ -40,6 +40,7 @@
import com.baidu.hugegraph.backend.store.BackendTable;
import com.baidu.hugegraph.backend.store.Shard;
import com.baidu.hugegraph.exception.NotFoundException;
import com.baidu.hugegraph.exception.NotSupportException;
import com.baidu.hugegraph.iterator.ExtendableIterator;
import com.baidu.hugegraph.type.HugeType;
import com.baidu.hugegraph.type.define.HugeKeys;
@@ -303,7 +304,7 @@ protected Clause relation2Cql(Relation relation) {
// return QueryBuilder.like(key, value);
case NEQ:
default:
throw new AssertionError("Unsupported relation: " + relation);
throw new NotSupportException("relation '%s'", relation);
}
}

@@ -28,7 +28,7 @@
import com.baidu.hugegraph.util.InsertionOrderUtil;
import com.google.common.collect.ImmutableSet;

public final class IdHolder {
public class IdHolder {

private final ConditionQuery query;
private final Function<ConditionQuery, PageIds> idsFetcher;
@@ -67,6 +67,7 @@ public IdHolder(ConditionQuery query,
}

public void merge(Set<Id> ids) {
E.checkNotNull(this.ids, "ids");
this.ids.addAll(ids);
}

@@ -24,7 +24,7 @@

import com.baidu.hugegraph.util.E;

public final class IdHolderList extends ArrayList<IdHolder> {
public class IdHolderList extends ArrayList<IdHolder> {

private static final long serialVersionUID = -738694176552424990L;

@@ -24,6 +24,7 @@

import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.backend.tx.AbstractTransaction.QueryResults;
import com.baidu.hugegraph.exception.NotSupportException;
import com.baidu.hugegraph.iterator.Metadatable;
import com.baidu.hugegraph.util.E;
@@ -32,15 +33,19 @@ public class PageEntryIterator implements Iterator<BackendEntry>, Metadatable {

private final QueryList queries;
private final long pageSize;
private QueryList.PageIterator results;
private PageInfo pageInfo;
private final PageInfo pageInfo;
private final QueryResults queryResults; // for upper layer

private QueryList.PageIterator pageResults;
private long remaining;

public PageEntryIterator(QueryList queries, long pageSize) {
this.queries = queries;
this.pageSize = pageSize;
this.results = QueryList.PageIterator.EMPTY;
this.pageInfo = this.parsePageState();
this.queryResults = new QueryResults(this);

this.pageResults = QueryList.PageIterator.EMPTY;
this.remaining = queries.parent().limit();
}

@@ -55,7 +60,7 @@ private PageInfo parsePageState() {

@Override
public boolean hasNext() {
if (this.results.iterator().hasNext()) {
if (this.pageResults.get().hasNext()) {
return true;
}
return this.fetch();
@@ -71,16 +76,17 @@ private boolean fetch() {
if (this.remaining != Query.NO_LIMIT && this.remaining < pageSize) {
pageSize = this.remaining;
}
this.results = this.queries.fetchNext(this.pageInfo, pageSize);
assert this.results != null;
this.pageResults = this.queries.fetchNext(this.pageInfo, pageSize);
assert this.pageResults != null;
this.queryResults.setQuery(this.pageResults.query());

if (this.results.iterator().hasNext()) {
if (!this.results.hasNextPage()) {
if (this.pageResults.get().hasNext()) {
if (!this.pageResults.hasNextPage()) {
this.pageInfo.increase();
} else {
this.pageInfo.page(this.results.page());
this.pageInfo.page(this.pageResults.page());
}
this.remaining -= this.results.total();
this.remaining -= this.pageResults.total();
return true;
} else {
this.pageInfo.increase();
@@ -93,7 +99,7 @@ public BackendEntry next() {
if (!this.hasNext()) {
throw new NoSuchElementException();
}
return this.results.iterator().next();
return this.pageResults.get().next();
}

@Override
@@ -106,4 +112,8 @@ public Object metadata(String meta, Object... args) {
}
throw new NotSupportException("Invalid meta '%s'", meta);
}

public QueryResults results() {
return this.queryResults;
}
}
@@ -26,30 +26,29 @@
import java.util.Set;
import java.util.function.Function;

import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;

import com.baidu.hugegraph.HugeGraph;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.query.IdQuery;
import com.baidu.hugegraph.backend.query.Query;
import com.baidu.hugegraph.backend.store.BackendEntry;
import com.baidu.hugegraph.backend.tx.AbstractTransaction.QueryResults;
import com.baidu.hugegraph.config.CoreOptions;
import com.baidu.hugegraph.iterator.FlatMapperIterator;
import com.baidu.hugegraph.util.Bytes;
import com.baidu.hugegraph.util.CollectionUtil;
import com.baidu.hugegraph.util.E;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

public final class QueryList {

private final HugeGraph graph;
private final Query parent;
// The size of each page fetched by the inner page
private final Function<Query, Iterator<BackendEntry>> fetcher;
private final Function<Query, QueryResults> fetcher;
private final List<QueryHolder> queries;

public QueryList(HugeGraph graph, Query parent,
Function<Query, Iterator<BackendEntry>> fetcher) {
Function<Query, QueryResults> fetcher) {
this.graph = graph;
this.parent = parent;
this.fetcher = fetcher;
@@ -60,7 +59,7 @@ protected Query parent() {
return this.parent;
}

protected Function<Query, Iterator<BackendEntry>> fetcher() {
protected Function<Query, QueryResults> fetcher() {
return this.fetcher;
}

@@ -93,21 +92,21 @@ public boolean empty() {
return this.queries.isEmpty();
}

public Iterator<BackendEntry> fetch() {
public QueryResults fetch() {
assert !this.queries.isEmpty();
if (this.parent.paging()) {
int pageSize = this.graph.configuration()
.get(CoreOptions.QUERY_PAGE_SIZE);
return new PageEntryIterator(this, pageSize);
} else {
return this.fetchAll();
PageEntryIterator iterator = new PageEntryIterator(this, pageSize);
/*
* NOTE: PageEntryIterator query will change every fetch time.
* TODO: sort results by input ids in each page.
*/
return iterator.results();
}
}

protected Iterator<BackendEntry> fetchAll() {
return new FlatMapperIterator<>(this.queries.iterator(), q -> {
return q.iterator();
});
// Fetch all results once
return QueryResults.flatMap(this.queries.iterator(), q -> q.iterator());
}

protected PageIterator fetchNext(PageInfo pageInfo, long pageSize) {
@@ -157,7 +156,7 @@ private interface QueryHolder {
* For non-paging situation
* @return BackendEntry iterator
*/
public Iterator<BackendEntry> iterator();
public QueryResults iterator();

/**
* For paging situation
@@ -183,7 +182,7 @@ public OptimizedQuery(Query query) {
}

@Override
public Iterator<BackendEntry> iterator() {
public QueryResults iterator() {
return fetcher().apply(this.query);
}

@@ -196,11 +195,11 @@ public PageIterator iterator(int index, String page, long pageSize) {
if (this.query.limit() == Query.NO_LIMIT) {
query.limit(pageSize);
}
Iterator<BackendEntry> iterator = fetcher().apply(query);
QueryResults results = fetcher().apply(query);
// Must iterate all entries before get the next page
List<BackendEntry> results = IteratorUtils.list(iterator);
return new PageIterator(results.iterator(),
PageInfo.pageState(iterator));
return new PageIterator(results.list().iterator(),
results.queries(),
PageInfo.pageState(results.iterator()));
}

@Override
@@ -222,23 +221,28 @@ public IndexQuery(List<IdHolder> holders) {
}

@Override
public Iterator<BackendEntry> iterator() {
return new FlatMapperIterator<>(this.holders.iterator(), holder -> {
if (holder.ids().isEmpty()) {
return null;
}
Set<Id> ids = holder.ids();
if (parent().limit() != Query.NO_LIMIT &&
ids.size() > parent().limit()) {
/*
* Avoid too many ids in one time query,
* Assume it will get one result by each id
*/
ids = CollectionUtil.subSet(ids, 0, (int) parent().limit());
}
IdQuery query = new IdQuery(parent(), ids);
return fetcher().apply(query);
});
public QueryResults iterator() {
if (this.holders.size() == 1) {
return this.each(this.holders.get(0));
}
return QueryResults.flatMap(this.holders.iterator(), this::each);
}

private QueryResults each(IdHolder holder) {
Set<Id> ids = holder.ids();
if (ids.isEmpty()) {
return null;
}
if (parent().limit() != Query.NO_LIMIT &&
ids.size() > parent().limit()) {
/*
* Avoid too many ids in one time query,
* Assume it will get one result by each id
*/
ids = CollectionUtil.subSet(ids, 0, (int) parent().limit());
}
IdQuery query = new IdQuery(parent(), ids);
return fetcher().apply(query);
}

@Override
@@ -249,7 +253,8 @@ public PageIterator iterator(int index, String page, long pageSize) {
return PageIterator.EMPTY;
}
IdQuery query = new IdQuery(parent(), pageIds.ids());
return new PageIterator(fetcher().apply(query),
QueryResults results = fetcher().apply(query);
return new PageIterator(results.iterator(), results.queries(),
pageIds.pageState());
}

@@ -263,18 +268,21 @@ public static class PageIterator {

public static final PageIterator EMPTY = new PageIterator(
Collections.emptyIterator(),
ImmutableList.of(Query.NONE),
PageState.EMPTY);

private final Iterator<BackendEntry> iterator;
private final List<Query> queries;
private final PageState pageState;

public PageIterator(Iterator<BackendEntry> iterator,
PageState pageState) {
List<Query> queries, PageState pageState) {
this.iterator = iterator;
this.queries = queries;
this.pageState = pageState;
}

public Iterator<BackendEntry> iterator() {
public Iterator<BackendEntry> get() {
return this.iterator;
}

@@ -283,6 +291,12 @@ public boolean hasNextPage() {
PageState.EMPTY_BYTES);
}

public Query query() {
E.checkState(this.queries.size() == 1,
"Expect queries size 1, but got: %s", this.queries);
return this.queries.get(0);
}

public String page() {
return this.pageState.toString();
}

0 comments on commit d47f309

Please sign in to comment.