MID-8842 upgrade, fixed setup database schema action. audit part stil…

…l doesn't work

config/sql/native-new/postgres-new-upgrade-audit.sql

@@ -28,8 +28,6 @@ Perhaps you have separate audit database?', current_database(), current_schema()
     END
 $$;
 
-;; -- sql statements delimiter, used by ninja
-
 -- SCHEMA-COMMIT 4.4: commit 69e8c29b
 
 -- changes for 4.4.1
@@ -93,8 +91,6 @@ BEGIN
 END $$;
 $aac$);
 
-;;
-
 -- SCHEMA-COMMIT 4.4.1: commit de18c14f
 
 -- changes for 4.5
@@ -104,8 +100,6 @@ call apply_audit_change(2, $aa$
 ALTER TYPE ObjectType ADD VALUE IF NOT EXISTS 'MESSAGE_TEMPLATE' AFTER 'LOOKUP_TABLE';
 $aa$);
 
-;;
-
 -- SCHEMA-COMMIT 4.6: commit 71f2df50
 
 -- changes for 4.7
@@ -115,8 +109,6 @@ call apply_audit_change(3, $aa$
    ALTER TYPE ObjectType ADD VALUE IF NOT EXISTS 'MARK' AFTER 'LOOKUP_TABLE';
 $aa$);
 
-;;
-
 -- WRITE CHANGES ABOVE ^^
 -- IMPORTANT: update apply_audit_change number at the end of postgres-new-audit.sql
 -- to match the number used in the last change here!

config/sql/native-new/postgres-new-upgrade.sql

@@ -29,8 +29,6 @@ Current database name is ''%'', schema name is ''%''.', current_database(), curr
     END
 $$;
 
-;;  -- sql statements delimiter, used by ninja
-
 -- SCHEMA-COMMIT 4.4: commit 20ad200b
 -- see: https://github.com/Evolveum/midpoint/blob/20ad200bd10a114fd70d2d131c0d11b5cd920150/config/sql/native-new/postgres-new.sql
 
@@ -61,8 +59,6 @@ CREATE TRIGGER m_org_mark_refresh_trunc_tr
     FOR EACH STATEMENT EXECUTE FUNCTION mark_org_closure_for_refresh_org();
 $aa$);
 
-;;
-
 -- SCHEMA-COMMIT 4.4.1: commit de18c14f
 
 -- changes for 4.5
@@ -73,8 +69,6 @@ call apply_change(2, $aa$
 ALTER TYPE ObjectType ADD VALUE IF NOT EXISTS 'MESSAGE_TEMPLATE' AFTER 'LOOKUP_TABLE';
 $aa$);
 
-;;
-
 call apply_change(3, $aa$
 CREATE TABLE m_message_template (
     oid UUID NOT NULL PRIMARY KEY REFERENCES m_object_oid(oid),
@@ -98,16 +92,12 @@ CREATE INDEX m_message_template_createTimestamp_idx ON m_message_template (creat
 CREATE INDEX m_message_template_modifyTimestamp_idx ON m_message_template (modifyTimestamp);
 $aa$);
 
-;;
-
 -- MID-7487 Identity matching
 -- We add the new enum value in separate change, because it must be committed before it is used.
 call apply_change(4, $aa$
 CREATE TYPE CorrelationSituationType AS ENUM ('UNCERTAIN', 'EXISTING_OWNER', 'NO_OWNER', 'ERROR');
 $aa$);
 
-;;
-
 call apply_change(5, $aa$
 ALTER TABLE m_shadow
 ADD COLUMN correlationStartTimestamp TIMESTAMPTZ,
@@ -122,8 +112,6 @@ CREATE INDEX m_shadow_correlationCaseOpenTimestamp_idx ON m_shadow (correlationC
 CREATE INDEX m_shadow_correlationCaseCloseTimestamp_idx ON m_shadow (correlationCaseCloseTimestamp);
 $aa$);
 
-;;
-
 -- SCHEMA-COMMIT 4.5: commit c5f19c9e
 
 -- changes for 4.6
@@ -134,24 +122,18 @@ call apply_change(6, $aa$
 CREATE TYPE AdministrativeAvailabilityStatusType AS ENUM ('MAINTENANCE', 'OPERATIONAL');
 $aa$);
 
-;;
-
 call apply_change(7, $aa$
 ALTER TABLE m_resource
 ADD COLUMN administrativeOperationalStateAdministrativeAvailabilityStatus AdministrativeAvailabilityStatusType;
 $aa$);
 
-;;
-
 -- smart correlation
 call apply_change(8, $aa$
 CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; -- fuzzy string match (levenshtein, etc.)
 
 ALTER TYPE ContainerType ADD VALUE IF NOT EXISTS 'FOCUS_IDENTITY' AFTER 'CASE_WORK_ITEM';
 $aa$);
 
-;;
-
 call apply_change(9, $aa$
 CREATE TABLE m_focus_identity (
     ownerOid UUID NOT NULL REFERENCES m_object_oid(oid) ON DELETE CASCADE,
@@ -170,22 +152,16 @@ ALTER TABLE m_focus ADD normalizedData JSONB;
 CREATE INDEX m_focus_normalizedData_idx ON m_focus USING gin(normalizedData);
 $aa$);
 
-;;
-
 -- resource templates
 call apply_change(10, $aa$
 ALTER TABLE m_resource ADD template BOOLEAN;
 $aa$);
 
-;;
-
 -- MID-8053: "Active" connectors detection
 call apply_change(11, $aa$
 ALTER TABLE m_connector ADD available BOOLEAN;
 $aa$);
 
-;;
-
 -- SCHEMA-COMMIT 4.5: commit c5f19c9e
 
 -- No changes for audit schema in 4.6
@@ -202,8 +178,6 @@ ALTER TYPE ObjectType ADD VALUE IF NOT EXISTS 'SIMULATION_RESULT' AFTER 'SHADOW'
 ALTER TYPE ContainerType ADD VALUE IF NOT EXISTS 'SIMULATION_RESULT_PROCESSED_OBJECT' AFTER 'OPERATION_EXECUTION';
 $aa$);
 
-;;
-
 -- Simulations, tables
 call apply_change(13, $aa$
 CREATE TABLE m_simulation_result (
@@ -336,8 +310,6 @@ CREATE INDEX m_ref_object_effective_mark_targetOidRelationId_idx
     ON m_ref_object_effective_mark (targetOid, relationId);
 $aa$);
 
-;;
-
 -- Minor index name fixes
 call apply_change(14, $aa$
 ALTER INDEX m_ref_object_create_approverTargetOidRelationId_idx
@@ -346,15 +318,11 @@ ALTER INDEX m_ref_object_modify_approverTargetOidRelationId_idx
     RENAME TO m_ref_object_modify_approver_targetOidRelationId_idx;
 $aa$);
 
-;;
-
 -- Making resource.abstract queryable
 call apply_change(15, $aa$
 ALTER TABLE m_resource ADD abstract BOOLEAN;
 $aa$);
 
-;;
-
 -- WRITE CHANGES ABOVE ^^
 -- IMPORTANT: update apply_change number at the end of postgres-new.sql
 -- to match the number used in the last change here!

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/SetupDatabaseSchemaAction.java

@@ -1,10 +1,16 @@
 package com.evolveum.midpoint.ninja.action;
 
 import java.io.File;
+import java.io.IOException;
 import java.sql.SQLException;
 import java.util.List;
 import javax.sql.DataSource;
 
+import com.evolveum.midpoint.common.configuration.api.MidpointConfiguration;
+import com.evolveum.midpoint.repo.sqale.SqaleRepositoryConfiguration;
+
+import com.evolveum.midpoint.repo.sqlbase.DataSourceFactory;
+
 import org.springframework.context.ApplicationContext;
 
 import com.evolveum.midpoint.ninja.opts.SetupDatabaseSchemaOptions;
@@ -14,26 +20,42 @@ public class SetupDatabaseSchemaAction extends RepositoryAction<SetupDatabaseSch
 
     @Override
     public void execute() throws Exception {
+        // this is manual setup of datasource for midpoint, can't be done via spring application context initialization with repository
+        // because sqale repository during initialization loads data from m_uri and m_ext_item (not yet existing)
         final ApplicationContext applicationContext = context.getApplicationContext();
 
-        final File scriptsDirectory = options.getScriptsDirectory();
+        final MidpointConfiguration midpointConfiguration = applicationContext.getBean(MidpointConfiguration.class);
+
+        SqaleRepositoryConfiguration repositoryConfiguration = new SqaleRepositoryConfiguration(
+                midpointConfiguration.getConfiguration(
+                        MidpointConfiguration.REPOSITORY_CONFIGURATION));
+        repositoryConfiguration.init();
+        DataSourceFactory dataSourceFactory = new DataSourceFactory(repositoryConfiguration);
+        try {
+
+            final File scriptsDirectory = options.getScriptsDirectory();
+
+            // upgrade midpoint repository
+            final DataSource dataSource = dataSourceFactory.createDataSource("ninja-repository");
+            executeScripts(dataSource, scriptsDirectory, options.getScripts());
 
-        // upgrade midpoint repository
-        final DataSource dataSource = applicationContext.getBean(DataSource.class);
-        executeScripts(dataSource, scriptsDirectory, options.getScripts());
+            // upgrade audit database
+            if (!options.isNoAudit()) {
+                // todo figure out how to initialize datasource for audit, it's a mess
 
-        // upgrade audit database
-        if (!options.isNoAudit()) {
-            final DataSource auditDataSource = NinjaUtils.getAuditDataSourceBean(applicationContext);
-            executeScripts(auditDataSource, scriptsDirectory, options.getAuditScripts());
+                final DataSource auditDataSource = NinjaUtils.getAuditDataSourceBean(applicationContext);
+                executeScripts(auditDataSource, scriptsDirectory, options.getAuditScripts());
+            }
+        } finally {
+            dataSourceFactory.destroy();
         }
     }
 
-    private void executeScripts(DataSource dataSource, File scriptsDirectory, List<File> scripts) throws SQLException {
+    private void executeScripts(DataSource dataSource, File scriptsDirectory, List<File> scripts) throws IOException, SQLException {
         List<File> files = scripts.stream()
                 .map(script -> scriptsDirectory != null ? new File(scriptsDirectory, script.getPath()) : script)
                 .toList();
 
-        NinjaUtils.executeSqlScripts(dataSource, files, options.getQuerySeparator());
+        NinjaUtils.executeSqlScripts(dataSource, files);
     }
 }

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/upgrade/step/DatabaseSchemaStep.java

@@ -56,11 +56,11 @@ public StepResult execute() throws Exception {
 
         // upgrade midpoint repository
         final DataSource dataSource = applicationContext.getBean(DataSource.class);
-        NinjaUtils.executeSqlScripts(dataSource, List.of(new File(distributionDirectory, MIDPOINT_DB_UPGRADE_FILE)), ";;");
+        NinjaUtils.executeSqlScripts(dataSource, List.of(new File(distributionDirectory, MIDPOINT_DB_UPGRADE_FILE)));
 
         // upgrade audit database
         final DataSource auditDataSource = NinjaUtils.getAuditDataSourceBean(applicationContext);
-        NinjaUtils.executeSqlScripts(auditDataSource, List.of(new File(distributionDirectory, AUDIT_DB_UPGRADE_FILE)), ";;");
+        NinjaUtils.executeSqlScripts(auditDataSource, List.of(new File(distributionDirectory, AUDIT_DB_UPGRADE_FILE)));
 
         return new StepResult() {
 

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/util/NinjaUtils.java

@@ -12,8 +12,12 @@
 import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
 import java.sql.SQLException;
+import java.sql.Statement;
 import java.text.DecimalFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 import javax.sql.DataSource;
@@ -22,11 +26,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 import org.springframework.context.ApplicationContext;
-import org.springframework.core.io.FileSystemResourceLoader;
-import org.springframework.core.io.Resource;
-import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator;
 
 import com.evolveum.midpoint.init.AuditServiceProxy;
 import com.evolveum.midpoint.ninja.impl.Command;
@@ -226,26 +226,20 @@ public static List<ObjectTypes> getTypes(Set<ObjectTypes> selected) {
         return types;
     }
 
-    public static void executeSqlScripts(@NotNull DataSource dataSource, @NotNull List<File> scripts, @Nullable String querySeparator) throws SQLException {
+    public static void executeSqlScripts(@NotNull DataSource dataSource, @NotNull List<File> scripts) throws IOException, SQLException {
         try (Connection connection = dataSource.getConnection()) {
             boolean autocommit = connection.getAutoCommit();
             connection.setAutoCommit(true);
 
             try {
-                ResourceDatabasePopulator populator = new ResourceDatabasePopulator();
-                if (querySeparator != null) {
-                    populator.setSeparator(querySeparator);
-                }
-                populator.setSqlScriptEncoding(StandardCharsets.UTF_8.name());
-
                 for (File script : scripts) {
-                    FileSystemResourceLoader loader = new FileSystemResourceLoader();
-                    Resource resource = loader.getResource("file:" + script.getAbsolutePath());
+                    Statement stmt = connection.createStatement();
 
-                    populator.addScript(resource);
-                }
+                    String sql = FileUtils.readFileToString(script, StandardCharsets.UTF_8);
+                    stmt.execute(sql);
 
-                populator.populate(connection);
+                    stmt.close();
+                }
             } finally {
                 connection.setAutoCommit(autocommit);
             }

tools/ninja/src/test/java/com/evolveum/midpoint/ninja/TestMain.java

@@ -6,12 +6,24 @@
  */
 package com.evolveum.midpoint.ninja;
 
+import com.evolveum.midpoint.ninja.util.NinjaUtils;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import org.apache.commons.io.FileUtils;
+import org.postgresql.Driver;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.sql.Statement;
+import java.util.List;
+
 /**
  * Created by Viliam Repan (lazyman).
  */
 public class TestMain {
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws Exception {
         //        jc.parse("-m", "./src/test/resources/midpoint-home",
 //                "export",
 //                "-O", "./export.xml",
@@ -33,10 +45,33 @@ public static void main(String[] args) {
 //                + "--temp-dir ../../_mess/mid8842/.upgrade-process")
 //                .split(" ");
 
-                input = "-h setup-database".split(" ");
+//        input = "-h setup-database".split(" ");
 
-//        input = "-v -m ../../_mess/create0 setup-database --scripts-directory ../../config/sql/native-new".split(" ");
+        input = "-v --offline -m ../../_mess/create0 setup-database --scripts-directory ../../config/sql/native-new".split(" "); // mo
 
         Main.main(input);
+
+//        HikariConfig config = new HikariConfig();
+//        config.setAutoCommit(true);
+//        config.setDriverClassName(Driver.class.getName());
+//        config.setJdbcUrl("jdbc:postgresql://localhost:5432/create1");
+//        config.setUsername("create1");
+//        config.setPassword("create1");
+//        config.setMaximumPoolSize(5);
+//        config.setMinimumIdle(1);
+//        HikariDataSource ds = new HikariDataSource(config);
+//
+//        try {
+//            Statement stmt = ds.getConnection().createStatement();
+//            stmt.execute(FileUtils.readFileToString(new File("/Users/lazyman/Work/monoted/git/evolveum/midpoint-support/config/sql/native-new/postgres-new.sql"), StandardCharsets.UTF_8));
+//
+////            NinjaUtils.executeSqlScripts(ds, List.of(
+////                    new File("/Users/lazyman/Work/monoted/git/evolveum/midpoint-support/config/sql/native-new/postgres-new.sql"),
+////                    new File("/Users/lazyman/Work/monoted/git/evolveum/midpoint-support/config/sql/native-new/postgres-new-quartz.sql"),
+////                    new File("/Users/lazyman/Work/monoted/git/evolveum/midpoint-support/config/sql/native-new/postgres-new-audit.sql")
+////            ), null);
+//        } finally {
+//            ds.close();
+//        }
     }
 }