Permalink
Browse files

readded mappings

  • Loading branch information...
wilhelmfrank committed Nov 26, 2018
1 parent a42291d commit 5112c52d28331ddb983b0f8e5b1dea270c8c749e
@@ -469,7 +469,7 @@
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parsers</artifactId>
<version>1.13</version>
<version>1.19.1</version>
<exclusions>
<exclusion>
<!-- Conflict with jersey mulitpart client (used by integration tests) -->
@@ -41,10 +41,14 @@ public Integer call() {
// ElasticInitializer.getIndexNameFromAliasName(ElasticService.DATA_ALIAS);
// final String index = ElasticInitializer.createIndex();
// addAllMappings(index);
for (final ElasticIndices index : ElasticIndices.values()) {
ElasticInitializer.initializeIndex(index);
new ElasticIndexer(index.name()).addMapping();
}
reindexUsers(ElasticIndices.users.name());
reindexUserGroups(ElasticIndices.usergroups.name());
reindexItems(ElasticIndices.items.name());
reindexContents(ElasticIndices.content.name());
// reindexContents(ElasticIndices.content.name());
reindexFolders(ElasticIndices.folders.name());
// ElasticInitializer.setNewIndexAndRemoveOldIndex(index);
LOGGER.info("Reindex done!");
@@ -338,7 +338,7 @@ public SearchResult search(SearchQuery searchQuery, SortCriterion sortCri, User
*/
public void reindex(String index) throws ImejiException {
LOGGER.info("Indexing collections...");
final ElasticIndexer indexer = new ElasticIndexer(index, ElasticIndices.folders);
final ElasticIndexer indexer = new ElasticIndexer(index);
final List<CollectionImeji> collections = retrieveAll();
indexer.indexBatch(collections);
LOGGER.info("collections reindexed!");
@@ -350,7 +350,7 @@ private boolean extractContent(ContentVO contentVO) {
*/
public void reindex(String index) throws ImejiException {
LOGGER.info("Indexing Content...");
final ElasticIndexer indexer = new ElasticIndexer(index, ElasticIndices.content);
final ElasticIndexer indexer = new ElasticIndexer(index);
final SearchServiceAbstract<ContentVO>.RetrieveIterator iterator = iterateAll(50);
LOGGER.info("+++ " + iterator.getSize() + " content to index +++");
int count = 0;
@@ -44,9 +44,8 @@
*/
public class MoveFacade implements Serializable {
private static final long serialVersionUID = 6158359006910656203L;
private final ElasticIndexer collectionIndexer = new ElasticIndexer(ElasticService.DATA_ALIAS,
ElasticIndices.folders);
private final ElasticIndexer itemsIndexer = new ElasticIndexer(ElasticService.DATA_ALIAS, ElasticIndices.items);
private final ElasticIndexer collectionIndexer = new ElasticIndexer(ElasticIndices.folders.name());
private final ElasticIndexer itemsIndexer = new ElasticIndexer(ElasticIndices.items.name());
private final CollectionService collectionService = new CollectionService();
private final MessageService messageService = new MessageService();
private final ItemService itemService = new ItemService();
@@ -48,9 +48,8 @@
private static final long serialVersionUID = 3966446108673573909L;
private final Authorization authorization = new Authorization();
private final WorkflowValidator workflowValidator = new WorkflowValidator();
private final ElasticIndexer collectionIndexer = new ElasticIndexer(ElasticService.DATA_ALIAS,
ElasticIndices.folders);
private final ElasticIndexer itemIndexer = new ElasticIndexer(ElasticService.DATA_ALIAS, ElasticIndices.items);
private final ElasticIndexer collectionIndexer = new ElasticIndexer(ElasticIndices.folders.name());
private final ElasticIndexer itemIndexer = new ElasticIndexer(ElasticIndices.items.name());

/**
* Release a collection and its item
@@ -594,7 +594,7 @@ public void withdraw(List<Item> l, String comment, User user) throws ImejiExcept
*/
public void reindex(String index) throws ImejiException {
LOGGER.info("Indexing Items...");
final ElasticIndexer indexer = new ElasticIndexer(index, ElasticIndices.items);
final ElasticIndexer indexer = new ElasticIndexer(ElasticIndices.items.name());
LOGGER.info("Retrieving Items...");
// (1) read item UIDs from Jena database (2) use item UIDs to read item content
// from Jena
@@ -9,6 +9,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
@@ -18,6 +19,7 @@
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Client;
@@ -59,7 +61,7 @@
private static final Logger LOGGER = LogManager.getLogger(ElasticIndexer.class);
private static final ObjectMapper mapper = new ObjectMapper();
private final String index;
private final String dataType;
private final String dataType = "_doc";
// private final ElasticAnalysers analyser;
private String mappingFile = "elasticsearch/Elastic_TYPE_Mapping.json";

@@ -76,11 +78,11 @@
* @param analyser
*/

public ElasticIndexer(String indexName, ElasticIndices dataType) {
public ElasticIndexer(String indexName) {
this.index = indexName;
this.dataType = dataType.name();
// this.dataType = dataType.name();
// this.analyser = analyser;
this.mappingFile = mappingFile.replace("_TYPE_", StringUtils.capitalize(this.dataType));
this.mappingFile = mappingFile.replace("_TYPE_", StringUtils.capitalize(this.index));
}

/**
@@ -352,17 +354,21 @@ private String getId(Object obj) {
/**
* Add a mapping to the fields (important to have a better search)
*/
/*
* public void addMapping() { try { final String jsonMapping = new
* String(Files.readAllBytes(
* Paths.get(ElasticIndexer.class.getClassLoader().getResource(mappingFile).
* toURI())), "UTF-8") .replace("XXX_ANALYSER_XXX", analyser.name());
* ElasticService.getClient().admin().indices().preparePutMapping(this.index).
* setType(dataType) .setSource(jsonMapping).execute().actionGet(); } catch
* (final Exception e) {
* LOGGER.error("Error initializing the Elastic Search Mapping " + mappingFile,
* e); } }
*/
public void addMapping() {
try {
final String jsonMapping = new String(
Files.readAllBytes(
Paths.get(ElasticIndexer.class.getClassLoader().getResource(mappingFile).toURI())),
"UTF-8");
PutMappingRequest request = new PutMappingRequest(this.index);
request.type(dataType);
request.source(jsonMapping, XContentType.JSON);
AcknowledgedResponse putMappingResponse = ElasticService.getClient().indices().putMapping(request,
RequestOptions.DEFAULT);
} catch (final Exception e) {
LOGGER.error("Error initializing the Elastic Search Mapping " + mappingFile, e);
}
}

@Override
public void updatePartial(String indexName, String id, Object obj) {
@@ -84,9 +84,8 @@ private static void start(String clusterName, RestHighLevelClient client) throws
// initializeIndex();
LOGGER.info("Add elasticsearch mappings...");
for (final ElasticIndices index : ElasticIndices.values()) {
// new ElasticIndexer(ElasticService.DATA_ALIAS, type,
// ElasticService.ANALYSER).addMapping();
initializeIndex(index);
new ElasticIndexer(index.name()).addMapping();
}
LOGGER.info("...done!");
}
@@ -195,12 +194,11 @@ public static String createIndex(String name) {

LOGGER.info("Creating a new index " + indexName);

final String settingsName = ElasticService.SETTINGS_DEFAULT;
final String settingsName = ElasticService.SETTINGS_DUCET;

final Path settingsJson = Paths
.get(ElasticIndexer.class.getClassLoader().getResource(settingsName).toURI());
HttpEntity entity = new InputStreamEntity(Files.newInputStream(settingsJson), 0,
ContentType.APPLICATION_JSON);
HttpEntity entity = new InputStreamEntity(Files.newInputStream(settingsJson), ContentType.APPLICATION_JSON);
Request req = new Request("PUT", "/" + indexName);
req.setEntity(entity);
Response resp = ElasticService.getClient().getLowLevelClient().performRequest(req);
@@ -69,7 +69,7 @@
public ElasticSearch(SearchObjectTypes... types) {
this.type = Stream.of(types).map(t -> ElasticIndices.toElasticIndex(t)).collect(Collectors.toList())
.toArray(new ElasticIndices[types.length]);
this.indexer = new ElasticIndexer(ElasticService.DATA_ALIAS, this.type[0]);
this.indexer = new ElasticIndexer(this.type[0].name());
}

@Override
@@ -20,7 +20,7 @@
static String CLUSTER_DIR = "null";
// public static ElasticAnalysers ANALYSER;
static final String SETTINGS_DEFAULT = "elasticsearch/Settings_default.json";
// static final String SETTINGS_DUCET = "elasticsearch/Settings_ducet.json";
static final String SETTINGS_DUCET = "elasticsearch/Settings_ducet.json";

/**
* The Index where all data are indexed
@@ -34,7 +34,7 @@
*
*/
public enum ElasticIndices {
content, items, folders, users, usergroups;
items, folders, users, usergroups;

/**
* Map a {@link SearchObjectTypes} to an ElasticIndex
@@ -52,8 +52,6 @@ public static ElasticIndices toElasticIndex(SearchObjectTypes type) {
return ElasticIndices.users;
case USERGROUPS :
return ElasticIndices.usergroups;
case CONTENT :
return ElasticIndices.content;
default :
return ElasticIndices.items;
}
@@ -759,7 +759,7 @@ private QueryBuilder allQuery(SearchPair pair) {
* @return
*/
private QueryBuilder contentQuery(QueryBuilder q) {
return JoinQueryBuilders.hasChildQuery(ElasticIndices.content.name(), q, null);
return JoinQueryBuilders.hasChildQuery(ElasticIndices.items.name(), q, null);
}

/**
@@ -135,7 +135,8 @@ public void delete(User user) throws ImejiException {
public User retrieve(String email, User user) throws ImejiException {
final SearchQuery query = new SearchQuery();
query.addPair(new SearchPair(SearchFields.email, SearchOperators.EQUALS, email, false));
final SearchResult result = search.search(query, null, Imeji.adminUser, null, 0, 1);
final SearchResult result = search.search(ElasticIndices.users.name(), query, null, Imeji.adminUser, null, 0,
1);
if (result.getNumberOfRecords() == 1) {
return controller.retrieve(URI.create(result.getResults().get(0)), user);
}
@@ -497,7 +498,7 @@ public static boolean adminUserExist() {

public void reindex(String index) throws ImejiException {
LOGGER.info("Indexing users...");
final ElasticIndexer indexer = new ElasticIndexer(index, ElasticIndices.users);
final ElasticIndexer indexer = new ElasticIndexer(ElasticIndices.users.name());
final List<User> users = retrieveAll();
LOGGER.info("+++ " + users.size() + " users to index +++");
indexer.indexBatch(users);
@@ -246,7 +246,7 @@ public void removeUserFromAllGroups(User userToRemove, User userRemover) throws
*/
public void reindex(String index) throws ImejiException {
LOGGER.info("Indexing users...");
final ElasticIndexer indexer = new ElasticIndexer(index, ElasticIndices.usergroups);
final ElasticIndexer indexer = new ElasticIndexer(index);
final List<UserGroup> groups = (List<UserGroup>) retrieveAll();
LOGGER.info("+++ " + groups.size() + " user groups to index +++");
indexer.indexBatch(groups);
@@ -1,5 +1,5 @@
{
"folders": {
"_doc": {
"dynamic": "strict",
"properties": {
"all": {
@@ -1,5 +1,5 @@
{
"items": {
"_doc": {
"dynamic": "strict",
"properties": {
"all": {
@@ -1,5 +1,5 @@
{
"usergroups": {
"_doc": {
"dynamic": "strict",
"properties": {
"all": {
@@ -1,5 +1,5 @@
{
"users": {
"_doc": {
"dynamic": "strict",
"properties": {
"all": {
@@ -1,30 +1,35 @@
{
"index": {
"analysis": {
"analyzer": {
"ngram_analyzer": {
"tokenizer": "ngram_tokenizer"
},
"edgegram_analyzer": {
"tokenizer": "edgegram_tokenizer",
"filter": [
"lowercase"
]
}
},
"tokenizer": {
"ngram_tokenizer": {
"type": "nGram",
"min_gram": "3",
"max_gram": "3"
},
"edgegram_tokenizer": {
"type": "edge_ngram",
"min_gram": 2,
"max_gram": 100,
"token_chars": ["letter", "digit", "punctuation", "symbol"]
}
}
}
}
"settings": {
"analysis": {
"analyzer": {
"ngram_analyzer": {
"tokenizer": "ngram_tokenizer"
},
"edgegram_analyzer": {
"tokenizer": "edgegram_tokenizer",
"filter": [
"lowercase"
]
}
},
"tokenizer": {
"ngram_tokenizer": {
"type": "nGram",
"min_gram": "3",
"max_gram": "3"
},
"edgegram_tokenizer": {
"type": "edge_ngram",
"min_gram": 2,
"max_gram": 100,
"token_chars": [
"letter",
"digit",
"punctuation",
"symbol"
]
}
}
}
}
}
@@ -1131,7 +1131,7 @@ index 899550446..201998dbb 100644
+ return search(indices[0], query, sortCri, user, folderUri, from, size, true);
}

- private SearchResult search(SearchQuery query, SortCriterion sortCri, User user, String folderUri,
- private SearchResult Fsearch(SearchQuery query, SortCriterion sortCri, User user, String folderUri,
+ private SearchResult search(String index, SearchQuery query, SortCriterion sortCri, User user, String folderUri,
int from, int size, boolean addFacets) {
size = size == -1 ? size = SEARCH_MAX_SIZE : size;

0 comments on commit 5112c52

Please sign in to comment.