Skip to content

Commit

Permalink
fix #2753 fix zero hit handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@marevol
marevol committed Jun 29, 2023
1 parent 47962a4 commit 833449c
Show file tree
Hide file tree
Showing 3 changed files with 272 additions and 7 deletions.
23 changes: 19 additions & 4 deletions src/main/java/org/codelibs/fess/rank/fusion/RankFusionProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ protected List<Map<String, Object>> searchWithMultipleSearchers(final String que
final var docs = scoreDocMap.values().stream()
.sorted((e1, e2) -> Float.compare(toFloat(e2.get(scoreField)), toFloat(e1.get(scoreField)))).toList();
int offset = 0;
for (int i = 0; i < windowSize / 2; i++) {
for (int i = 0; i < windowSize / 2 && i < docs.size(); i++) {
if (!mainIdSet.contains(docs.get(i).get(idField))) {
offset++;
}
Expand All @@ -193,9 +193,24 @@ protected List<Map<String, Object>> searchWithMultipleSearchers(final String que
if (Relation.EQUAL_TO.toString().equals(mainResult.getAllRecordCountRelation())) {
allRecordCount += offset;
}
return createResponseList(docs.subList(startPosition, startPosition + pageSize), allRecordCount,
mainResult.getAllRecordCountRelation(), mainResult.getQueryTime(), mainResult.isPartialResults(),
mainResult.getFacetResponse(), startPosition, pageSize, offset);
return createResponseList(extractList(docs, pageSize, startPosition), allRecordCount, mainResult.getAllRecordCountRelation(),
mainResult.getQueryTime(), mainResult.isPartialResults(), mainResult.getFacetResponse(), startPosition, pageSize, offset);
}

protected List<Map<String, Object>> extractList(final List<Map<String, Object>> docs, final int pageSize, final int startPosition) {
final int size = docs.size();
if (size == 0) {
return docs; // empty
}
int fromIndex = startPosition;
if (fromIndex >= size) {
fromIndex = size - 1;
}
int toIndex = startPosition + pageSize;
if (toIndex >= size) {
toIndex = size;
}
return docs.subList(fromIndex, toIndex);
}

protected List<Map<String, Object>> searchWithMainSearcher(final String query, final SearchRequestParams params,
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/org/codelibs/fess/rank/fusion/SearchResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,13 @@ public static SearchResultBuilder create() {
return new SearchResultBuilder();
}

@Override
public String toString() {
return "SearchResult [documentList=" + documentList + ", allRecordCount=" + allRecordCount + ", allRecordCountRelation="
+ allRecordCountRelation + ", queryTime=" + queryTime + ", partialResults=" + partialResults + ", facetResponse="
+ facetResponse + "]";
}

static class SearchResultBuilder {

private long allRecordCount;
Expand Down
249 changes: 246 additions & 3 deletions src/test/java/org/codelibs/fess/rank/fusion/RankFusionProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public void test_default_1000docs_10size() throws Exception {
}
}

public void test_1searcher10_1000docs_100size() throws Exception {
public void test_1searcher10_45_45_1000docs_100size() throws Exception {
String query = "*";
int allRecordCount = 1000;
int pageSize = 100;
Expand Down Expand Up @@ -191,7 +191,7 @@ public void test_1searcher10_1000docs_100size() throws Exception {

if (rankFusionProcessor.search(query, new TestSearchRequestParams(1000, pageSize, offset),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(pageSize, list.size());
assertEquals(offset, list.size());
assertEquals(allRecordCount + offset, list.getAllRecordCount());
assertEquals(11, list.getAllPageCount());
assertEquals(1045, list.getCurrentEndRecordNumber());
Expand All @@ -208,6 +208,249 @@ public void test_1searcher10_1000docs_100size() throws Exception {
}
}

public void test_1searcher0_0docs_100size() throws Exception {
String query = "*";
int allRecordCount = 0;
int pageSize = 100;
int offset = 0;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(0, 0, 0));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(0, list.size());
assertEquals(0, list.getAllRecordCount());
assertEquals(1, list.getAllPageCount());
assertEquals(0, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(0, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher0_10docs_100size() throws Exception {
String query = "*";
int allRecordCount = 10;
int pageSize = 100;
int offset = 0;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(0, 0, 0));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(10, list.size());
assertEquals(10, list.getAllRecordCount());
assertEquals(1, list.getAllPageCount());
assertEquals(10, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher0_1000docs_100size() throws Exception {
String query = "*";
int allRecordCount = 1000;
int pageSize = 100;
int offset = 0;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(0, 0, 0));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(pageSize, list.size());
assertEquals(1000, list.getAllRecordCount());
assertEquals(10, list.getAllPageCount());
assertEquals(100, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher10_0docs_100size() throws Exception {
String query = "*";
int allRecordCount = 0;
int pageSize = 100;
int offset = 10;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(10, 0, 0));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(10, list.size());
assertEquals(10, list.getAllRecordCount());
assertEquals(1, list.getAllPageCount());
assertEquals(10, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher10_10docs_100size() throws Exception {
String query = "*";
int allRecordCount = 10;
int pageSize = 100;
int offset = 0;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(10, 0, 0));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(10, list.size());
assertEquals(10, list.getAllRecordCount());
assertEquals(1, list.getAllPageCount());
assertEquals(10, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher10_1000docs_100size() throws Exception {
String query = "*";
int allRecordCount = 1000;
int pageSize = 100;
int offset = 0;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(10, 0, 0));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(pageSize, list.size());
assertEquals(1000, list.getAllRecordCount());
assertEquals(10, list.getAllPageCount());
assertEquals(100, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher1000_0docs_100size() throws Exception {
String query = "*";
int allRecordCount = 0;
int pageSize = 100;
int offset = 100;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(0, 0, 1000));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(pageSize, list.size());
assertEquals(100, list.getAllRecordCount());
assertEquals(1, list.getAllPageCount());
assertEquals(100, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher1000_10docs_100size() throws Exception {
String query = "*";
int allRecordCount = 10;
int pageSize = 100;
int offset = 90;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(0, 0, 1000));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(pageSize, list.size());
assertEquals(100, list.getAllRecordCount());
assertEquals(1, list.getAllPageCount());
assertEquals(100, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

public void test_1searcher1000_1000docs_100size() throws Exception {
String query = "*";
int allRecordCount = 1000;
int pageSize = 100;
int offset = 50;
try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) {
rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount));
rankFusionProcessor.register(new TestSubSearcher(0, 0, 1000));
rankFusionProcessor.init();

if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0),
OptionalThing.empty()) instanceof QueryResponseList list) {
assertEquals(pageSize, list.size());
assertEquals(1050, list.getAllRecordCount());
assertEquals(11, list.getAllPageCount());
assertEquals(100, list.getCurrentEndRecordNumber());
assertEquals(1, list.getCurrentPageNumber());
assertEquals(1, list.getCurrentStartRecordNumber());
assertEquals(offset, list.getOffset());
assertEquals(100, list.getPageSize());
assertEquals(0, list.getStart());
} else {
fail();
}
}
}

static class TestMainSearcher extends RankFusionSearcher {

private long allRecordCount;
Expand All @@ -221,7 +464,7 @@ protected SearchResult search(String query, SearchRequestParams params, Optional
int start = params.getStartPosition();
int size = params.getPageSize();
SearchResultBuilder builder = SearchResult.create();
for (int i = start; i < start + size; i++) {
for (int i = start; i < start + size && i < allRecordCount; i++) {
Map<String, Object> doc = new HashMap<>();
doc.put(ID_FIELD, Integer.toString(i));
doc.put("score", 1.0f / (i + 1));
Expand Down

0 comments on commit 833449c

Please sign in to comment.