[NAE-2136] Speed up Elasticsearch reindex

pom.xml

@@ -64,9 +64,10 @@
         <drools.version>7.70.0.Final</drools.version>
         <sonar.organization>netgrif-oss</sonar.organization>
         <sonar.host.url>https://sonarcloud.io</sonar.host.url>
+        <jackson.version>2.15.0-rc1</jackson.version>
     </properties>
 
-<!--    <repositories>-->
+    <repositories>
 <!--        <repository>-->
 <!--            <id>oss.snapshots</id>-->
 <!--            <name>OSSRH SNAPSHOT</name>-->
@@ -90,7 +91,17 @@
 <!--            <id>mulesoft</id>-->
 <!--            <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>-->
 <!--        </repository>-->
-<!--    </repositories>-->
+        <repository>
+            <id>jitpack.io</id>
+            <url>https://jitpack.io</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
 
     <dependencyManagement>
         <dependencies>
@@ -352,12 +363,11 @@
             <version>${querydsl.version}</version>
         </dependency>
 
-
         <!-- QR code -->
         <dependency>
-            <groupId>com.github.kenglxn.qrgen</groupId>
-            <artifactId>javase</artifactId>
-            <version>2.6.0</version>
+            <groupId>com.github.kenglxn</groupId>
+            <artifactId>QRGen</artifactId>
+            <version>3.0.1</version>
         </dependency>
 
         <!-- ElasticSearch -->
@@ -366,6 +376,20 @@
             <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
         </dependency>
 
+        <!-- Java 8 Date/time -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.glassfish</groupId>
+            <artifactId>jakarta.json</artifactId>
+            <version>2.0.1</version>
+        </dependency>
+
+
         <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
         <dependency>
             <groupId>com.google.protobuf</groupId>
@@ -395,13 +419,6 @@
             <version>${drools.version}</version>
         </dependency>
 
-        <!-- QRGen -->
-        <dependency>
-            <groupId>net.glxn.qrgen</groupId>
-            <artifactId>core</artifactId>
-            <version>2.0</version>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
@@ -492,38 +509,38 @@
         <dependency>
             <groupId>com.fasterxml.jackson</groupId>
             <artifactId>jackson-base</artifactId>
-            <version>2.15.0-rc1</version>
+            <version>${jackson.version}</version>
             <type>pom</type>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
-            <version>2.15.0-rc1</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.15.0-rc1</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.jaxrs</groupId>
             <artifactId>jackson-jaxrs-json-provider</artifactId>
-            <version>2.15.0-rc1</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>2.15.0-rc1</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-xml</artifactId>
-            <version>2.15.0-rc1</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.module</groupId>
             <artifactId>jackson-module-jsonSchema</artifactId>
-            <version>2.15.0-rc1</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>io.minio</groupId>

src/main/java/com/netgrif/application/engine/configuration/ElasticsearchConfiguration.java

@@ -1,25 +1,34 @@
 package com.netgrif.application.engine.configuration;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.netgrif.application.engine.configuration.properties.ElasticsearchProperties;
 import com.netgrif.application.engine.configuration.properties.UriProperties;
+import com.netgrif.application.engine.elastic.serializer.LocalDateTimeJsonDeserializer;
+import com.netgrif.application.engine.elastic.serializer.LocalDateTimeJsonSerializer;
 import com.netgrif.application.engine.workflow.service.CaseEventHandler;
+import lombok.RequiredArgsConstructor;
 import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
 import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
 
+import java.time.LocalDateTime;
+
 @Configuration
+@RequiredArgsConstructor
 public class ElasticsearchConfiguration {
 
-    @Value("${spring.data.elasticsearch.url}")
-    private String url;
-
-    @Value("${spring.data.elasticsearch.searchport}")
-    private int port;
-
     @Value("${spring.data.elasticsearch.index.petriNet}")
     private String petriNetIndex;
 
@@ -32,11 +41,9 @@ public class ElasticsearchConfiguration {
     @Value("${spring.data.elasticsearch.reindex}")
     private String cron;
 
-    private final UriProperties uriProperties;
+    private final ElasticsearchProperties elasticsearchProperties;
 
-    public ElasticsearchConfiguration(UriProperties uriProperties) {
-        this.uriProperties = uriProperties;
-    }
+    private final UriProperties uriProperties;
 
     @Bean
     public String springElasticsearchReindex() {
@@ -65,9 +72,18 @@ public String elasticUriIndex() {
 
     @Bean
     public RestHighLevelClient client() {
-
-        return new RestHighLevelClient(
-                RestClient.builder(new HttpHost(url, port, "http")));
+        RestClientBuilder builder = RestClient.builder(new HttpHost(elasticsearchProperties.getUrl(), elasticsearchProperties.getSearchPort()));
+        if (hasCredentials()) {
+            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY,
+                    new UsernamePasswordCredentials(
+                            elasticsearchProperties.getUsername(),
+                            elasticsearchProperties.getPassword()
+                    )
+            );
+            builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+        }
+        return new RestHighLevelClient(builder);
     }
 
     @Bean
@@ -79,4 +95,22 @@ public ElasticsearchOperations elasticsearchTemplate() {
     public CaseEventHandler caseEventHandler() {
         return new CaseEventHandler();
     }
+
+    @Bean(name = "elasticCaseObjectMapper")
+    public ObjectMapper configureMapper() {
+        ObjectMapper mapper = new ObjectMapper();
+
+        JavaTimeModule javaTimeModule = new JavaTimeModule();
+        javaTimeModule.addSerializer(LocalDateTime.class, new LocalDateTimeJsonSerializer());
+        javaTimeModule.addDeserializer(LocalDateTime.class, new LocalDateTimeJsonDeserializer());
+
+        mapper.registerModule(javaTimeModule);
+        mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        return mapper;
+    }
+
+    private boolean hasCredentials() {
+        return elasticsearchProperties.getUsername() != null && !elasticsearchProperties.getUsername().isBlank() &&
+                elasticsearchProperties.getPassword() != null && !elasticsearchProperties.getPassword().isBlank();
+    }
 }

src/main/java/com/netgrif/application/engine/configuration/properties/ElasticsearchProperties.java

@@ -6,6 +6,8 @@
 import org.springframework.stereotype.Component;
 
 import javax.annotation.PostConstruct;
+import javax.validation.Valid;
+import javax.validation.constraints.Min;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -33,6 +35,10 @@ public class ElasticsearchProperties {
 
     private String url;
 
+    private String username;
+
+    private String password;
+
     private Map<String, String> index;
 
     private boolean analyzerEnabled = false;
@@ -53,6 +59,9 @@ public class ElasticsearchProperties {
 
     private List<String> defaultSearchFilters = new ArrayList<>();
 
+    @Valid
+    private BatchProperties batch = new BatchProperties();
+
     @PostConstruct
     public void init() {
         indexSettings.putIfAbsent("max_result_window", 10000000);
@@ -72,4 +81,13 @@ public void init() {
     public Map<String, Object> getClassSpecificSettings(String className) {
         return classSpecificIndexSettings.getOrDefault(className, new HashMap<>());
     }
+
+    @Data
+    public static class BatchProperties {
+        @Min(1)
+        private int caseBatchSize = 5000;
+
+        @Min(1)
+        private int taskBatchSize = 20000;
+    }
 }

src/main/java/com/netgrif/application/engine/elastic/domain/CaseField.java

@@ -0,0 +1,24 @@
+package com.netgrif.application.engine.elastic.domain;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import org.springframework.data.elasticsearch.annotations.Field;
+
+import java.util.List;
+
+import static org.springframework.data.elasticsearch.annotations.FieldType.*;
+
+@Data
+@NoArgsConstructor
+@EqualsAndHashCode(callSuper = true)
+public class CaseField extends DataField {
+
+    @Field(type = Text)
+    private List<String> caseValue;
+
+    public CaseField(List<String> value) {
+        super(value.toString());
+        this.caseValue = value;
+    }
+}

src/main/java/com/netgrif/application/engine/elastic/domain/ElasticCase.java

@@ -1,9 +1,5 @@
 package com.netgrif.application.engine.elastic.domain;
 
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
-import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
 import com.netgrif.application.engine.workflow.domain.Case;
 import com.netgrif.application.engine.workflow.domain.TaskPair;
 import lombok.AllArgsConstructor;
@@ -18,6 +14,7 @@
 
 import java.sql.Timestamp;
 import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -56,8 +53,6 @@ public class ElasticCase {
 
     private String title;
 
-    @JsonSerialize(using = LocalDateTimeSerializer.class)
-    @JsonDeserialize(using = LocalDateTimeDeserializer.class)
     @Field(type = FieldType.Date, format = DateFormat.date_hour_minute_second_millis)
     private LocalDateTime creationDate;
 
@@ -121,7 +116,7 @@ public ElasticCase(Case useCase) {
         processId = useCase.getPetriNetId();
         visualId = useCase.getVisualId();
         title = useCase.getTitle();
-        creationDate = useCase.getCreationDate();
+        creationDate = useCase.getCreationDate().truncatedTo(ChronoUnit.MILLIS);
         creationDateSortable = Timestamp.valueOf(useCase.getCreationDate()).getTime();
         author = useCase.getAuthor().getId();
         authorName = useCase.getAuthor().getFullName();

src/main/java/com/netgrif/application/engine/elastic/serializer/LocalDateTimeJsonDeserializer.java

@@ -0,0 +1,29 @@
+package com.netgrif.application.engine.elastic.serializer;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+
+public class LocalDateTimeJsonDeserializer extends JsonDeserializer<LocalDateTime> {
+    private static final DateTimeFormatter FORMATTER = new DateTimeFormatterBuilder()
+            .appendPattern("yyyy-MM-dd'T'HH:mm:ss")
+            .optionalStart()
+            .appendFraction(ChronoField.MILLI_OF_SECOND, 1, 3, true)
+            .optionalEnd()
+            .toFormatter();
+
+    @Override
+    public LocalDateTime deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+        String value = p.getValueAsString();
+        if (value == null || value.isEmpty()) {
+            return null;
+        }
+        return LocalDateTime.parse(value, FORMATTER);
+    }
+}

src/main/java/com/netgrif/application/engine/elastic/serializer/LocalDateTimeJsonSerializer.java

@@ -0,0 +1,18 @@
+package com.netgrif.application.engine.elastic.serializer;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
+public class LocalDateTimeJsonSerializer extends JsonSerializer<LocalDateTime> {
+    private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
+
+    @Override
+    public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
+        gen.writeString(FORMATTER.format(value));
+    }
+}