perform: use prepared batch statement for fast reliable DB inserts

excel-importer-test/src_test/com/axonivy/util/excel/test/TestEntityDataLoader.java

@@ -2,13 +2,11 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import java.io.IOException;
 import java.nio.file.Path;
 import java.util.List;
 
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
-import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -32,7 +30,7 @@ public class TestEntityDataLoader {
   private IIvyEntityManager unit;
 
   @Test
-  void loadDataToEntity(@TempDir Path dir) throws IOException, CoreException {
+  void loadDataToEntity(@TempDir Path dir) throws Exception {
     Path path = dir.resolve("customers.xlsx");
     TstRes.loadTo(path, "sample.xlsx");
 
@@ -51,9 +49,9 @@ void loadDataToEntity(@TempDir Path dir) throws IOException, CoreException {
       customer.getResource().delete(true, new NullProgressMonitor());
     }
   }
-  
+
   @Test
-  void loadArznei(@TempDir Path dir) throws IOException, CoreException {
+  void loadArznei(@TempDir Path dir) throws Exception {
     Path path = dir.resolve("meds.xlsx");
     TstRes.loadTo(path, "ArzneimittelLight.xlsx");
 

excel-importer/META-INF/MANIFEST.MF

@@ -19,5 +19,6 @@ Require-Bundle: ch.ivyteam.ivy.scripting.dataclass,
  ch.ivyteam.lib.apache.commons.lang3,
  javax.persistence,
  ch.ivyteam.ivy.dialog.config,
- ch.ivyteam.ivy.dialog.jsf.ui
+ ch.ivyteam.ivy.dialog.jsf.ui,
+ ch.ivyteam.lib.hibernate.entitymanager
 Bundle-ActivationPolicy: lazy

excel-importer/src/com/axonivy/util/excel/importer/EntityDataLoader.java

@@ -1,18 +1,24 @@
 package com.axonivy.util.excel.importer;
 
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 import javax.persistence.EntityManager;
-import javax.persistence.EntityTransaction;
-import javax.persistence.Query;
 
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.CellType;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.NullProgressMonitor;
+import org.hibernate.engine.jdbc.connections.spi.JdbcConnectionAccess;
+import org.hibernate.internal.SessionImpl;
 
 import ch.ivyteam.ivy.java.IJavaConfigurationManager;
 import ch.ivyteam.ivy.process.data.persistence.IIvyEntityManager;
@@ -31,47 +37,54 @@ public EntityDataLoader(IIvyEntityManager manager) {
     this.manager = manager;
   }
 
-  public void load(Sheet sheet, IEntityClass entity) {
+  public void load(Sheet sheet, IEntityClass entity) throws SQLException {
+    load(sheet, entity, new NullProgressMonitor());
+  }
+
+  public void load(Sheet sheet, IEntityClass entity, IProgressMonitor monitor) throws SQLException {
     Iterator<Row> rows = sheet.rowIterator();
     rows.next(); // skip header
+    monitor.beginTask("Importing Excel data rows", sheet.getLastRowNum());
 
     List<? extends IEntityClassField> fields = entity.getFields();
     var query = buildInsertQuery(entity, fields);
 
     EntityManager em = manager.createEntityManager();
-    EntityTransaction tx = em.getTransaction();
-    tx.begin();
+    JdbcConnectionAccess access = em.unwrap(SessionImpl.class).getJdbcConnectionAccess();
+    Connection con = access.obtainConnection();
     AtomicInteger rCount = new AtomicInteger();
     try {
+      var stmt = con.prepareStatement(query, Statement.NO_GENERATED_KEYS);
       rows.forEachRemaining(row -> {
-        Query insert = em.createNativeQuery(query);
-        rCount.incrementAndGet();
-        insertCallValuesAsParameter(fields, row, insert);
         try {
-          var inserted = insert.executeUpdate();
-          System.out.println("updateded "+inserted+" records");
-        } catch (Exception ex) {
-          LOGGER.error("Failed to insert "+insert);
+        rCount.incrementAndGet();
+          insertCallValuesAsParameter(fields, row, stmt);
+          stmt.addBatch();
+        } catch (SQLException ex) {
+          throw new RuntimeException(ex);
         }
       });
+      stmt.executeBatch();
+      con.commit();
+    } catch (Exception ex) {
+      LOGGER.error("warn "+ex);
     } finally {
       System.out.println("inserted " + rCount + " rows");
-      tx.commit();
+      access.releaseConnection(con);
       em.close();
     }
   }
 
-  private void insertCallValuesAsParameter(List<? extends IEntityClassField> fields, Row row, Query insert) {
+  private void insertCallValuesAsParameter(List<? extends IEntityClassField> fields, Row row, PreparedStatement stmt) throws SQLException {
     int c = 0;
     for(var field : fields) {
       if (field.getName().equals("id")) {
         continue;
       }
-      String name = field.getName();
       Cell cell = row.getCell(c);
       Object value = getValue(cell);
-      insert.setParameter(name, value);
       c++;
+      stmt.setObject(c, value);
     }
   }
 

excel-importer/src/com/axonivy/util/excel/importer/wizard/ExcelImportProcessor.java

@@ -1,6 +1,5 @@
 package com.axonivy.util.excel.importer.wizard;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Path;
@@ -86,7 +85,7 @@ public void run(IProgressMonitor monitor) throws InvocationTargetException {
         var manager = IDataClassManager.instance().getProjectDataModelFor(selectedSourceProject.getProject());
         try {
           importExcel(manager, importFile, m);
-        } catch (IOException ex) {
+        } catch (Exception ex) {
           status = EclipseUtil.createErrorStatus(ex);
         }
       }, null, IWorkspace.AVOID_UPDATE, progress);
@@ -97,7 +96,7 @@ public void run(IProgressMonitor monitor) throws InvocationTargetException {
     }
   }
 
-  private void importExcel(IProjectDataClassManager manager, FileResource excel, IProgressMonitor monitor) throws IOException {
+  private void importExcel(IProjectDataClassManager manager, FileResource excel, IProgressMonitor monitor) throws Exception {
     Workbook wb = null;
     try(InputStream is = excel.read().inputStream()) {
       wb = ExcelLoader.load(excel.name(), excel.read().inputStream());