Issue 24681 allow to run new data upgrade tasks (#24710)

* #24681 Adding support to run data upgrade tasks

* #24681 Adding integration tests

* #24681 Applying code style

dotCMS/src/integration-test/java/com/dotcms/MainSuite.java

@@ -164,6 +164,7 @@
 import com.dotmarketing.quartz.job.IntegrityDataGenerationJobTest;
 import com.dotmarketing.quartz.job.PopulateContentletAsJSONJobTest;
 import com.dotmarketing.quartz.job.StartEndScheduledExperimentsJobTest;
+import com.dotmarketing.startup.StartupTasksExecutorDataTest;
 import com.dotmarketing.startup.StartupTasksExecutorTest;
 import com.dotmarketing.startup.runalways.Task00050LoadAppsSecretsTest;
 import com.dotmarketing.startup.runonce.Task05195CreatesDestroyActionAndAssignDestroyDefaultActionsToTheSystemWorkflowTest;
@@ -647,6 +648,7 @@
         PopulateContentletAsJSONJobTest.class,
         ContentTypeDestroyAPIImplTest.class,
         Task230328AddMarkedForDeletionColumnTest.class,
+        StartupTasksExecutorDataTest.class,
 //        AnalyticsAPIImplTest.class,
 //        AccessTokenRenewJobTest.class,
 })

dotCMS/src/integration-test/java/com/dotmarketing/startup/StartupTasksExecutorDataTest.java

@@ -0,0 +1,114 @@
+package com.dotmarketing.startup;
+
+import com.dotcms.IntegrationTestBase;
+import com.dotcms.business.CloseDBIfOpened;
+import com.dotcms.util.IntegrationTestInitService;
+import com.dotmarketing.common.db.DotConnect;
+import com.dotmarketing.exception.DotDataException;
+import com.dotmarketing.util.Logger;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class StartupTasksExecutorDataTest extends IntegrationTestBase {
+
+    @BeforeClass
+    public static void prepare() throws Exception {
+        //Setting web app environment
+        IntegrationTestInitService.getInstance().init();
+    }
+
+    /**
+     * Drops the db_version table
+     */
+    private void dropDBVersionTable() {
+
+        try {
+            final DotConnect dotConnect = new DotConnect();
+            dotConnect.setSQL("DROP TABLE IF EXISTS db_version");
+            dotConnect.loadResult();
+        } catch (DotDataException e) {
+            Logger.error(this, "Error dropping db_version table", e);
+            fail(String.format("Error dropping db_version table: %s", e.getMessage()));
+        }
+    }
+
+    /**
+     * Drops the data_version table
+     */
+    private void dropDataVersionTable() {
+
+        try {
+            final DotConnect dotConnect = new DotConnect();
+            dotConnect.setSQL("DROP TABLE IF EXISTS data_version");
+            dotConnect.loadResult();
+        } catch (DotDataException e) {
+            Logger.error(this, "Error dropping data_version table", e);
+            fail(String.format("Error dropping data_version table: %s", e.getMessage()));
+        }
+    }
+
+    /**
+     * <b>Method to test:</b> StartupTasksExecutor.getInstance().insureDataVersionTable()
+     * <p>
+     * <b>Given scenario:</b> Drop the data_version table, run the StartupTasksExecutor.getInstance().insureDataVersionTable()
+     * <p>
+     * <b>Expected result:</b> The data_version table should exist
+     */
+    @Test
+    @CloseDBIfOpened
+    public void testStartupTasksExecutorGetDataVersionTable() throws Exception {
+
+        // Drop the data_version table
+        dropDataVersionTable();
+
+        // Create the table
+        var executor = StartupTasksExecutor.getInstance();
+        executor.insureDataVersionTable();
+
+        try {
+            // Checking the latest version
+            var dataVersion = executor.currentDataVersion();
+            assertEquals(0, dataVersion);
+        } catch (Exception e) {
+            Logger.error(this, "Error checking latest version in data_version table", e);
+            fail(String.format("Error checking latest version in data_version table: %s", e.getMessage()));
+        }
+    }
+
+    /**
+     * <b>Method to test:</b> StartupTasksExecutor.getInstance().executeDataUpgrades()
+     * <p>
+     * <b>Given scenario:</b> Drop the data_version table, run the StartupTasksExecutor.getInstance().insureDataVersionTable(),
+     * to finally execute the data upgrades.
+     * <p>
+     * <b>Expected result:</b> The data_version table should exist and the data_version table current version should be greater
+     * than 1
+     */
+    @Test
+    @CloseDBIfOpened
+    public void testStartupTasksExecutorExecuteDataUpgrades() throws Exception {
+
+        // Drop the version tables
+        dropDataVersionTable();
+        dropDBVersionTable();
+
+        // Create the table
+        var executor = StartupTasksExecutor.getInstance();
+        executor.insureDataVersionTable();
+
+        // Run the data upgrades
+        executor.executeDataUpgrades();
+
+        try {
+            // Checking the latest version
+            var dataVersion = executor.currentDataVersion();
+            assertTrue(dataVersion > 1);
+        } catch (Exception e) {
+            Logger.error(this, "Error checking latest version in data_version table", e);
+            fail(String.format("Error checking latest version in data_version table: %s", e.getMessage()));
+        }
+    }
+
+}

dotCMS/src/main/java/com/dotmarketing/startup/StartupTasksExecutor.java

@@ -17,32 +17,34 @@
 
 public class StartupTasksExecutor {
 
-
 	private static StartupTasksExecutor executor;
 
+    private final String pgCreate  = "CREATE TABLE db_version (db_version integer NOT NULL, date_update timestamp with time zone NOT NULL, CONSTRAINT db_version_pkey PRIMARY KEY (db_version));";
+    private final String pgCreateDataVersion  = "CREATE TABLE data_version (data_version integer NOT NULL, date_update timestamp with time zone NOT NULL, CONSTRAINT data_version_pkey PRIMARY KEY (data_version));";
+    private final String myCreate  = "CREATE TABLE `db_version` (`db_version` INTEGER UNSIGNED NOT NULL,`date_update` DATETIME NOT NULL, PRIMARY KEY (`db_version`))";
+    private final String myCreateDataVersion  = "CREATE TABLE `data_version` (`data_version` INTEGER UNSIGNED NOT NULL,`date_update` DATETIME NOT NULL, PRIMARY KEY (`data_version`))";
+    private final String oraCreate = "CREATE TABLE \"DB_VERSION\" ( \"DB_VERSION\" INTEGER NOT NULL , \"DATE_UPDATE\" TIMESTAMP NOT NULL, PRIMARY KEY (\"DB_VERSION\") )";
+    private final String oraCreateDataVersion = "CREATE TABLE \"DATA_VERSION\" ( \"DATA_VERSION\" INTEGER NOT NULL , \"DATE_UPDATE\" TIMESTAMP NOT NULL, PRIMARY KEY (\"DATA_VERSION\") )";
+    private final String msCreate  = "CREATE TABLE db_version (	db_version int NOT NULL , date_update datetime NOT NULL, PRIMARY KEY (db_version) )";
+    private final String msCreateDataVersion  = "CREATE TABLE data_version ( data_version int NOT NULL , date_update datetime NOT NULL, PRIMARY KEY (data_version) )";
 
-
-
-	private final String pgCreate  = "CREATE TABLE db_version (db_version integer NOT NULL, date_update timestamp with time zone NOT NULL, CONSTRAINT db_version_pkey PRIMARY KEY (db_version));";
-	private final String myCreate  = "CREATE TABLE `db_version` (`db_version` INTEGER UNSIGNED NOT NULL,`date_update` DATETIME NOT NULL, PRIMARY KEY (`db_version`))";
-	private final String oraCreate = "CREATE TABLE \"DB_VERSION\" ( \"DB_VERSION\" INTEGER NOT NULL , \"DATE_UPDATE\" TIMESTAMP NOT NULL, PRIMARY KEY (\"DB_VERSION\") )";
-	private final String msCreate  = "CREATE TABLE db_version (	db_version int NOT NULL , date_update datetime NOT NULL, PRIMARY KEY (db_version) )";
-
-	private final String SELECT = "SELECT max(db_version) AS test FROM db_version";
-	private final String INSERT = "INSERT INTO db_version (db_version,date_update) VALUES (?,?)";
+    private final String SELECT = "SELECT max(db_version) AS test FROM db_version";
+    private final String SELECT_DATA_VERSION = "SELECT max(data_version) AS test FROM data_version";
+    private final String INSERT = "INSERT INTO db_version (db_version,date_update) VALUES (?,?)";
+    private final String INSERT_DATA_VERSION = "INSERT INTO data_version (data_version,date_update) VALUES (?,?)";
 
 	private static final Pattern TASK_ID_PATTERN = Pattern.compile("[0-9]+");
 
 	final boolean firstTimeStart;
-
-
-	private StartupTasksExecutor() {
 
-	    insureDbVersionTable();
-	    Config.DB_VERSION = currentDbVersion();
+    private StartupTasksExecutor() {
+
+        insureDbVersionTable();
+        insureDataVersionTable();
+        Config.DB_VERSION = currentDbVersion();
+        Config.DATA_VERSION = currentDataVersion();
         this.firstTimeStart = (Config.DB_VERSION==0);
-
-	}
+    }
 
 	public static synchronized StartupTasksExecutor getInstance() {
 		if (executor == null)
@@ -52,16 +54,30 @@ public static synchronized StartupTasksExecutor getInstance() {
 
 
 	private final String createTableSQL() {
-	    
-	       return (DbConnectionFactory.isPostgres()) 
+
+	       return (DbConnectionFactory.isPostgres())
 	                        ? pgCreate
-	                        : DbConnectionFactory.isMySql() 
-	                            ? myCreate 
+	                        : DbConnectionFactory.isMySql()
+	                            ? myCreate
 	                            : DbConnectionFactory.isOracle()
 	                                ? oraCreate
 	                                : msCreate;
 	}
 
+    /**
+     * Returns the SQL to create the data_version table
+     */
+    private final String createDataVersionTableSQL() {
+
+        return (DbConnectionFactory.isPostgres())
+                ? pgCreateDataVersion
+                : DbConnectionFactory.isMySql()
+                        ? myCreateDataVersion
+                        : DbConnectionFactory.isOracle()
+                                ? oraCreateDataVersion
+                                : msCreateDataVersion;
+    }
+
     /**
      * This will create the db version table if it does not already exist
      * @return
@@ -77,7 +93,22 @@ private boolean insureDbVersionTable() {
         }
 
     }
-
+
+    /**
+     * This will create the data version table if it does not already exist
+     */
+    @VisibleForTesting
+    boolean insureDataVersionTable() {
+
+        try {
+            currentDataVersion();
+            return true;
+        } catch (Exception e) {
+            return createDataVersionTable();
+        }
+
+    }
+
     /**
      * Runs with a separate DB connection
      * @return
@@ -91,7 +122,21 @@ private int currentDbVersion() {
             throw new DotRuntimeException(e);
         }
     }
-
+
+    /**
+     * Returns the current data version. Runs with a separate DB connection
+     */
+    @VisibleForTesting
+    int currentDataVersion() {
+        try (Connection conn = DbConnectionFactory.getDataSource().getConnection()) {
+            DotConnect db = new DotConnect().setSQL(SELECT_DATA_VERSION);
+            return db.loadInt("test");
+
+        } catch (Exception e) {
+            throw new DotRuntimeException(e);
+        }
+    }
+
     /**
      * Runs with a separate DB connection
      * @return
@@ -109,10 +154,25 @@ private boolean createDbVersionTable() {
         }
 
     }
-
-
+
+    /**
+     * Creates the data version table. Runs with a separate DB connection
+     */
+    private boolean createDataVersionTable() {
+
+        try (Connection conn = DbConnectionFactory.getDataSource().getConnection()) {
+            new DotConnect().setSQL(createDataVersionTableSQL()).loadResult(conn);
+            new DotConnect().setSQL(INSERT_DATA_VERSION).addParam(0).addParam(new Date()).loadResult(conn);
+            return true;
+
+        } catch (Exception e) {
+            Logger.debug(this.getClass(), e.getMessage(),e);
+            throw new DotRuntimeException(e);
+        }
+    }
+
     public void executeStartUpTasks() throws DotDataException {
-        
+
         Logger.debug(this.getClass(), "Running Startup Tasks");
 
 
@@ -167,9 +227,9 @@ String getTaskId(final String taskName){
         }
         return "-1";
     }
-    
-    
-    public void executeUpgrades() throws DotDataException {
+
+
+    public void executeSchemaUpgrades() throws DotDataException {
 
         Logger.info(this, "---");
         Logger.info(this, "");
@@ -227,7 +287,71 @@ public void executeUpgrades() throws DotDataException {
 
 
     }
-
+
+    /**
+     * Runs the data upgrade tasks, those that are not related to the database schema upgrade tasks,
+     * data tasks are mostly tasks to solve data issues using our existing APIs.
+     * <p>
+     * The list of data upgrade tasks is obtained from the
+     * {@link TaskLocatorUtil#getStartupRunOnceDataTaskClasses()}
+     *
+     * @throws DotDataException
+     */
+    public void executeDataUpgrades() throws DotDataException {
+
+        Logger.info(this, "---");
+        Logger.info(this, "");
+        Logger.info(this, "Running Data Upgrade Tasks");
+        Logger.info(this, "Database data version: " + Config.DATA_VERSION);
+
+        String name;
+
+        for (Class<?> c : TaskLocatorUtil.getStartupRunOnceDataTaskClasses()) {
+
+            name = c.getCanonicalName();
+            name = name.substring(name.lastIndexOf(".") + 1);
+            String id = getTaskId(name);
+
+            try {
+                int taskId = Integer.parseInt(id);
+                if (StartupTask.class.isAssignableFrom(c) && taskId > Config.DATA_VERSION) {
+                    StartupTask task;
+                    try {
+                        task = (StartupTask) c.newInstance();
+                    } catch (Exception e) {
+                        throw new DotRuntimeException(e.getMessage(), e);
+                    }
+
+                    if (!firstTimeStart && task.forceRun()) {
+                        HibernateUtil.startTransaction();
+                        Logger.info(this, "Running Data Upgrade Tasks: " + name);
+                        task.executeUpgrade();
+                    }
+
+                    new DotConnect()
+                            .setSQL(INSERT_DATA_VERSION)
+                            .addParam(taskId)
+                            .addParam(new Date())
+                            .loadResult();
+                    Logger.info(this, "Data upgraded to version: " + taskId);
+                    HibernateUtil.closeAndCommitTransaction();
+                    Config.DATA_VERSION = taskId;
+                }
+            } catch (Exception e) {
+                HibernateUtil.rollbackTransaction();
+                if (Config.getBooleanProperty("SYSTEM_EXIT_ON_STARTUP_FAILURE", true)) {
+                    Logger.error(this, "FATAL: " + e.getMessage(), e);
+                    System.exit(1);
+                }
+            } finally {
+                HibernateUtil.closeAndCommitTransaction();
+            }
+
+        }
+
+        Logger.info(this, "Finishing data upgrade tasks.");
+    }
+
 	/**
      * This will execute all the UT that were backported to the LTS version.
      * Will be run everytime the server gets restarted just like the Startup Tasks.
@@ -247,7 +371,7 @@ public void executeBackportedTasks() throws DotDataException {
                 name = c.getCanonicalName();
                 name = name.substring(name.lastIndexOf(".") + 1);
 		String id = getTaskId(name);
-                int taskId = Integer.parseInt(id); 
+                int taskId = Integer.parseInt(id);
 		if (StartupTask.class.isAssignableFrom(c) && taskId > Config.DB_VERSION) {
                     StartupTask  task = (StartupTask) c.newInstance();
                     if (task.forceRun()) {

dotCMS/src/main/java/com/dotmarketing/util/Config.java

@@ -64,6 +64,7 @@ public class Config {
     public static final String DOTCMS_USEWATCHERMODE = "dotcms.usewatchermode";
     public static final String USE_CONFIG_TEST_OVERRIDE_TRACKER = "USE_CONFIG_TEST_OVERRIDE_TRACKER";
     public static int DB_VERSION = 0;
+    public static int DATA_VERSION = 0;
 
     //Object Config properties      n
     public static javax.servlet.ServletContext CONTEXT = null;