From 289d263075152d24d91d5fd1db5f93d75272d4b4 Mon Sep 17 00:00:00 2001 From: Peter Nowak Date: Fri, 25 Jan 2019 16:54:11 +0100 Subject: [PATCH] DATAES-525 - Fixed issue in scrolling not applying the search query. The root cause of the deletion problem was the doScroll which did not apply the given query and therefor returned all entries from the index. The doScroll implementation has been fixed to apply the query and multiple unit tests have been added to ensure that the delete methods delete the desired documents only and leave the rest untouched. Also added unit tests for the scrolling to test it against real queries (not only matchAll). --- .../core/ElasticsearchRestTemplate.java | 6 + .../core/ElasticsearchTemplateTests.java | 168 ++++++++++++++++++ 2 files changed, 174 insertions(+) diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java index 5c0034c85b..fed118127d 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java @@ -978,6 +978,12 @@ private SearchResponse doScroll(SearchRequest request, SearchQuery searchQuery) Assert.notNull(searchQuery.getTypes(), "No type define for Query"); Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll"); + if (searchQuery.getQuery() != null) { + request.source().query(searchQuery.getQuery()); + } else { + request.source().query(QueryBuilders.matchAllQuery()); + } + if (searchQuery.getFilter() != null) { request.source().postFilter(searchQuery.getFilter()); } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java index 4fe02bff0f..c436562fa1 100755 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; + import org.apache.commons.lang.StringUtils; import org.assertj.core.util.Lists; import org.elasticsearch.action.get.MultiGetItemResponse; @@ -2363,6 +2365,172 @@ public void shouldReturnMappingForGivenEntityClass() { assertThat(((Map) ((Map) mapping.get("properties")).get("message")).get("type"), Matchers.is("text")); } + @Test // DATAES-525 + public void shouldDeleteOnlyDocumentsMatchedByDeleteQuery() { + List indexQueries = new ArrayList<>(); + + // given + // document to be deleted + String documentIdToDelete = UUID.randomUUID().toString(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(documentIdToDelete).message("some message") + .version(System.currentTimeMillis()).build())); + + // remaining document + String remainingDocumentId = UUID.randomUUID().toString(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(remainingDocumentId).message("some other message") + .version(System.currentTimeMillis()).build())); + elasticsearchTemplate.bulkIndex(indexQueries); + elasticsearchTemplate.refresh(SampleEntity.class); + + // when + DeleteQuery deleteQuery = new DeleteQuery(); + deleteQuery.setQuery(idsQuery().addIds(documentIdToDelete)); + elasticsearchTemplate.delete(deleteQuery, SampleEntity.class); + elasticsearchTemplate.refresh(SampleEntity.class); + + // then + // document with id "remainingDocumentId" should still be indexed + SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build(); + Page sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, SampleEntity.class); + assertThat(sampleEntities.getTotalElements(), equalTo(1L)); + assertThat(sampleEntities.getContent().get(0).getId(), is(remainingDocumentId)); + } + + @Test // DATAES-525 + public void shouldDeleteOnlyDocumentsMatchedByCriteriaQuery() { + List indexQueries = new ArrayList<>(); + + // given + // document to be deleted + String documentIdToDelete = UUID.randomUUID().toString(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(documentIdToDelete).message("some message") + .version(System.currentTimeMillis()).build())); + + // remaining document + String remainingDocumentId = UUID.randomUUID().toString(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(remainingDocumentId).message("some other message") + .version(System.currentTimeMillis()).build())); + elasticsearchTemplate.bulkIndex(indexQueries); + elasticsearchTemplate.refresh(SampleEntity.class); + + // when + CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("id").is(documentIdToDelete)); + elasticsearchTemplate.delete(criteriaQuery, SampleEntity.class); + elasticsearchTemplate.refresh(SampleEntity.class); + + // then + // document with id "remainingDocumentId" should still be indexed + SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build(); + Page sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, SampleEntity.class); + assertThat(sampleEntities.getTotalElements(), equalTo(1L)); + assertThat(sampleEntities.getContent().get(0).getId(), is(remainingDocumentId)); + } + + @Test // DATAES-525 + public void shouldDeleteDocumentForGivenIdOnly() { + List indexQueries = new ArrayList<>(); + + // given + // document to be deleted + String documentIdToDelete = UUID.randomUUID().toString(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(documentIdToDelete).message("some message") + .version(System.currentTimeMillis()).build())); + + // remaining document + String remainingDocumentId = UUID.randomUUID().toString(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(remainingDocumentId).message("some other message") + .version(System.currentTimeMillis()).build())); + elasticsearchTemplate.bulkIndex(indexQueries); + elasticsearchTemplate.refresh(SampleEntity.class); + + // when + elasticsearchTemplate.delete(SampleEntity.class, documentIdToDelete); + elasticsearchTemplate.refresh(SampleEntity.class); + + // then + // document with id "remainingDocumentId" should still be indexed + SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build(); + Page sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, SampleEntity.class); + assertThat(sampleEntities.getTotalElements(), equalTo(1L)); + assertThat(sampleEntities.getContent().get(0).getId(), is(remainingDocumentId)); + } + + @Test // DATAES-525 + public void shouldApplyCriteriaQueryToScanAndScrollForGivenCriteriaQuery() { + // given + List indexQueries = new ArrayList<>(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString()) + .message("some message that should be found by the scroll query").version(System.currentTimeMillis()) + .build())); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString()) + .message("some other message that should be found by the scroll query") + .version(System.currentTimeMillis()).build())); + String notFindableMessage = "this entity must not be found by the scroll query"; + indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString()) + .message(notFindableMessage).version(System.currentTimeMillis()).build())); + + elasticsearchTemplate.bulkIndex(indexQueries); + elasticsearchTemplate.refresh(SampleEntity.class); + + // when + CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("message").contains("message")); + criteriaQuery.addIndices(INDEX_NAME); + criteriaQuery.addTypes(TYPE_NAME); + criteriaQuery.setPageable(PageRequest.of(0, 10)); + + ScrolledPage scroll = (ScrolledPage) elasticsearchTemplate.startScroll(1000, + criteriaQuery, SampleEntity.class); + List sampleEntities = new ArrayList<>(); + while (scroll.hasContent()) { + sampleEntities.addAll(scroll.getContent()); + scroll = (ScrolledPage) elasticsearchTemplate.continueScroll(scroll.getScrollId(), 1000, + SampleEntity.class); + } + elasticsearchTemplate.clearScroll(scroll.getScrollId()); + + // then + assertThat(sampleEntities.size(), is(equalTo(2))); + assertThat(sampleEntities.stream().map(SampleEntity::getMessage).collect(Collectors.toList()), + not(contains(notFindableMessage))); + } + + @Test // DATAES-525 + public void shouldApplySearchQueryToScanAndScrollForGivenSearchQuery() { + // given + List indexQueries = new ArrayList<>(); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString()) + .message("some message that should be found by the scroll query").version(System.currentTimeMillis()) + .build())); + indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString()) + .message("some other message that should be found by the scroll query") + .version(System.currentTimeMillis()).build())); + String notFindableMessage = "this entity must not be found by the scroll query"; + indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString()) + .message(notFindableMessage).version(System.currentTimeMillis()).build())); + + elasticsearchTemplate.bulkIndex(indexQueries); + elasticsearchTemplate.refresh(SampleEntity.class); + + // when + SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchQuery("message", "message")) + .withIndices(INDEX_NAME).withTypes(TYPE_NAME).withPageable(PageRequest.of(0, 10)).build(); + + ScrolledPage scroll = (ScrolledPage) elasticsearchTemplate.startScroll(1000, + searchQuery, SampleEntity.class); + List sampleEntities = new ArrayList<>(); + while (scroll.hasContent()) { + sampleEntities.addAll(scroll.getContent()); + scroll = (ScrolledPage) elasticsearchTemplate.continueScroll(scroll.getScrollId(), 1000, + SampleEntity.class); + } + elasticsearchTemplate.clearScroll(scroll.getScrollId()); + + // then + assertThat(sampleEntities.size(), is(equalTo(2))); + assertThat(sampleEntities.stream().map(SampleEntity::getMessage).collect(Collectors.toList()), + not(contains(notFindableMessage))); + } + private IndexQuery getIndexQuery(SampleEntity sampleEntity) { return new IndexQueryBuilder() .withId(sampleEntity.getId())