Skip to content

Commit

Permalink
add es reindexer
Browse files Browse the repository at this point in the history
  • Loading branch information
ruddell committed Dec 5, 2019
1 parent f3611be commit 7ab7510
Show file tree
Hide file tree
Showing 2 changed files with 159 additions and 0 deletions.
@@ -0,0 +1,117 @@
package com.mycompany.myapp.service;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.github.vanroy.springdata.jest.JestElasticsearchTemplate;
import com.mycompany.myapp.domain.User;
import com.mycompany.myapp.repository.UserRepository;
import com.mycompany.myapp.repository.search.UserSearchRepository;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.persistence.ManyToMany;
import java.beans.IntrospectionException;
import java.beans.PropertyDescriptor;
import java.io.Serializable;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;

@Service
@Transactional(readOnly = true)
public class ElasticsearchIndexService {

private static final Lock reindexLock = new ReentrantLock();

private final Logger log = LoggerFactory.getLogger(ElasticsearchIndexService.class);

private final UserRepository userRepository;

private final UserSearchRepository userSearchRepository;

private final JestElasticsearchTemplate jestElasticsearchTemplate;

public ElasticsearchIndexService(UserRepository userRepository, UserSearchRepository userSearchRepository, JestElasticsearchTemplate jestElasticsearchTemplate) {
this.userRepository = userRepository;
this.userSearchRepository = userSearchRepository;
this.jestElasticsearchTemplate = jestElasticsearchTemplate;
}

@Async
public void reindexAll() {
if (reindexLock.tryLock()) {
try {
reindexForClass(User.class, userRepository, userSearchRepository);
log.info("Elasticsearch: Successfully performed reindexing");
} finally {
reindexLock.unlock();
}
} else {
log.info("Elasticsearch: concurrent reindexing attempt");
}
}

private <T, ID extends Serializable> void reindexForClass(Class<T> entityClass, JpaRepository<T, ID> jpaRepository,
ElasticsearchRepository<T, ID> elasticsearchRepository) {
jestElasticsearchTemplate.deleteIndex(entityClass);
try {
jestElasticsearchTemplate.createIndex(entityClass);
} catch (ResourceAlreadyExistsException e) {
// Do nothing. Index was already concurrently recreated by some other service.
}
jestElasticsearchTemplate.putMapping(entityClass);
if (jpaRepository.count() > 0) {
// if a JHipster entity field is the owner side of a many-to-many relationship, it should be loaded manually
List<Method> relationshipGetters = Arrays.stream(entityClass.getDeclaredFields())
.filter(field -> field.getType().equals(Set.class))
.filter(field -> field.getAnnotation(ManyToMany.class) != null)
.filter(field -> field.getAnnotation(ManyToMany.class).mappedBy().isEmpty())
.filter(field -> field.getAnnotation(JsonIgnore.class) == null)
.map(field -> {
try {
return new PropertyDescriptor(field.getName(), entityClass).getReadMethod();
} catch (IntrospectionException e) {
log.error("Error retrieving getter for class {}, field {}. Field will NOT be indexed",
entityClass.getSimpleName(), field.getName(), e);
return null;
}
})
.filter(Objects::nonNull)
.collect(Collectors.toList());

int size = 100;
for (int i = 0; i <= jpaRepository.count() / size; i++) {
Pageable page = new PageRequest(i, size);

This comment has been minimized.

Copy link
@geyuqiu

geyuqiu Jul 1, 2020

Pageable page = PageRequest.of(i, size);

log.info("Indexing page {} of {}, size {}", i, jpaRepository.count() / size, size);
Page<T> results = jpaRepository.findAll(page);
results.map(result -> {
// if there are any relationships to load, do it now
relationshipGetters.forEach(method -> {
try {
// eagerly load the relationship set
((Set) method.invoke(result)).size();
} catch (Exception ex) {
log.error(ex.getMessage());
}
});
return result;
});
elasticsearchRepository.saveAll(results.getContent());
}
}
log.info("Elasticsearch: Indexed all rows for {}", entityClass.getSimpleName());
}
}
@@ -0,0 +1,42 @@
package com.mycompany.myapp.web.rest;

import com.mycompany.myapp.security.AuthoritiesConstants;
import com.mycompany.myapp.security.SecurityUtils;
import com.mycompany.myapp.service.ElasticsearchIndexService;

import java.net.URISyntaxException;

import io.github.jhipster.web.util.HeaderUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.access.annotation.Secured;
import org.springframework.web.bind.annotation.*;
import org.springframework.http.ResponseEntity;

/**
* REST controller for managing Elasticsearch index.
*/
@RestController
@RequestMapping("/api")
public class ElasticsearchIndexResource {

private final Logger log = LoggerFactory.getLogger(ElasticsearchIndexResource.class);

private final ElasticsearchIndexService elasticsearchIndexService;

public ElasticsearchIndexResource(ElasticsearchIndexService elasticsearchIndexService) {
this.elasticsearchIndexService = elasticsearchIndexService;
}
/**
* POST /elasticsearch/index -> Reindex all Elasticsearch documents
*/
@PostMapping("/elasticsearch/index")
@Secured(AuthoritiesConstants.ADMIN)
public ResponseEntity<Void> reindexAll() throws URISyntaxException {
log.info("REST request to reindex Elasticsearch by user : {}", SecurityUtils.getCurrentUserLogin());
elasticsearchIndexService.reindexAll();
return ResponseEntity.accepted()
.headers(HeaderUtil.createAlert("mono", "elasticsearch.reindex.accepted", null))
.build();
}
}

0 comments on commit 7ab7510

Please sign in to comment.