+ * Given scenario: Drop the data_version table, run the StartupTasksExecutor.getInstance().insureDataVersionTable() + *
+ * Expected result: The data_version table should exist + */ + @Test + @CloseDBIfOpened + public void testStartupTasksExecutorGetDataVersionTable() throws Exception { + + // Drop the data_version table + dropDataVersionTable(); + + // Create the table + var executor = StartupTasksExecutor.getInstance(); + executor.insureDataVersionTable(); + + try { + // Checking the latest version + var dataVersion = executor.currentDataVersion(); + assertEquals(0, dataVersion); + } catch (Exception e) { + Logger.error(this, "Error checking latest version in data_version table", e); + fail(String.format("Error checking latest version in data_version table: %s", e.getMessage())); + } + } + + /** + * Method to test: StartupTasksExecutor.getInstance().executeDataUpgrades() + *
+ * Given scenario: Drop the data_version table, run the StartupTasksExecutor.getInstance().insureDataVersionTable(), + * to finally execute the data upgrades. + *
+ * Expected result: The data_version table should exist and the data_version table current version should be greater + * than 1 + */ + @Test + @CloseDBIfOpened + public void testStartupTasksExecutorExecuteDataUpgrades() throws Exception { + + // Drop the version tables + dropDataVersionTable(); + dropDBVersionTable(); + + // Create the table + var executor = StartupTasksExecutor.getInstance(); + executor.insureDataVersionTable(); + + // Run the data upgrades + executor.executeDataUpgrades(); + + try { + // Checking the latest version + var dataVersion = executor.currentDataVersion(); + assertTrue(dataVersion > 1); + } catch (Exception e) { + Logger.error(this, "Error checking latest version in data_version table", e); + fail(String.format("Error checking latest version in data_version table: %s", e.getMessage())); + } + } + +} \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotmarketing/startup/StartupTasksExecutor.java b/dotCMS/src/main/java/com/dotmarketing/startup/StartupTasksExecutor.java index b05270fa5908..860ccf2a8de1 100644 --- a/dotCMS/src/main/java/com/dotmarketing/startup/StartupTasksExecutor.java +++ b/dotCMS/src/main/java/com/dotmarketing/startup/StartupTasksExecutor.java @@ -17,32 +17,34 @@ public class StartupTasksExecutor { - private static StartupTasksExecutor executor; + private final String pgCreate = "CREATE TABLE db_version (db_version integer NOT NULL, date_update timestamp with time zone NOT NULL, CONSTRAINT db_version_pkey PRIMARY KEY (db_version));"; + private final String pgCreateDataVersion = "CREATE TABLE data_version (data_version integer NOT NULL, date_update timestamp with time zone NOT NULL, CONSTRAINT data_version_pkey PRIMARY KEY (data_version));"; + private final String myCreate = "CREATE TABLE `db_version` (`db_version` INTEGER UNSIGNED NOT NULL,`date_update` DATETIME NOT NULL, PRIMARY KEY (`db_version`))"; + private final String myCreateDataVersion = "CREATE TABLE `data_version` (`data_version` INTEGER UNSIGNED NOT NULL,`date_update` DATETIME NOT NULL, PRIMARY KEY (`data_version`))"; + private final String oraCreate = "CREATE TABLE \"DB_VERSION\" ( \"DB_VERSION\" INTEGER NOT NULL , \"DATE_UPDATE\" TIMESTAMP NOT NULL, PRIMARY KEY (\"DB_VERSION\") )"; + private final String oraCreateDataVersion = "CREATE TABLE \"DATA_VERSION\" ( \"DATA_VERSION\" INTEGER NOT NULL , \"DATE_UPDATE\" TIMESTAMP NOT NULL, PRIMARY KEY (\"DATA_VERSION\") )"; + private final String msCreate = "CREATE TABLE db_version ( db_version int NOT NULL , date_update datetime NOT NULL, PRIMARY KEY (db_version) )"; + private final String msCreateDataVersion = "CREATE TABLE data_version ( data_version int NOT NULL , date_update datetime NOT NULL, PRIMARY KEY (data_version) )"; - - - private final String pgCreate = "CREATE TABLE db_version (db_version integer NOT NULL, date_update timestamp with time zone NOT NULL, CONSTRAINT db_version_pkey PRIMARY KEY (db_version));"; - private final String myCreate = "CREATE TABLE `db_version` (`db_version` INTEGER UNSIGNED NOT NULL,`date_update` DATETIME NOT NULL, PRIMARY KEY (`db_version`))"; - private final String oraCreate = "CREATE TABLE \"DB_VERSION\" ( \"DB_VERSION\" INTEGER NOT NULL , \"DATE_UPDATE\" TIMESTAMP NOT NULL, PRIMARY KEY (\"DB_VERSION\") )"; - private final String msCreate = "CREATE TABLE db_version ( db_version int NOT NULL , date_update datetime NOT NULL, PRIMARY KEY (db_version) )"; - - private final String SELECT = "SELECT max(db_version) AS test FROM db_version"; - private final String INSERT = "INSERT INTO db_version (db_version,date_update) VALUES (?,?)"; + private final String SELECT = "SELECT max(db_version) AS test FROM db_version"; + private final String SELECT_DATA_VERSION = "SELECT max(data_version) AS test FROM data_version"; + private final String INSERT = "INSERT INTO db_version (db_version,date_update) VALUES (?,?)"; + private final String INSERT_DATA_VERSION = "INSERT INTO data_version (data_version,date_update) VALUES (?,?)"; private static final Pattern TASK_ID_PATTERN = Pattern.compile("[0-9]+"); final boolean firstTimeStart; - - - private StartupTasksExecutor() { - insureDbVersionTable(); - Config.DB_VERSION = currentDbVersion(); + private StartupTasksExecutor() { + + insureDbVersionTable(); + insureDataVersionTable(); + Config.DB_VERSION = currentDbVersion(); + Config.DATA_VERSION = currentDataVersion(); this.firstTimeStart = (Config.DB_VERSION==0); - - } + } public static synchronized StartupTasksExecutor getInstance() { if (executor == null) @@ -52,16 +54,30 @@ public static synchronized StartupTasksExecutor getInstance() { private final String createTableSQL() { - - return (DbConnectionFactory.isPostgres()) + + return (DbConnectionFactory.isPostgres()) ? pgCreate - : DbConnectionFactory.isMySql() - ? myCreate + : DbConnectionFactory.isMySql() + ? myCreate : DbConnectionFactory.isOracle() ? oraCreate : msCreate; } + /** + * Returns the SQL to create the data_version table + */ + private final String createDataVersionTableSQL() { + + return (DbConnectionFactory.isPostgres()) + ? pgCreateDataVersion + : DbConnectionFactory.isMySql() + ? myCreateDataVersion + : DbConnectionFactory.isOracle() + ? oraCreateDataVersion + : msCreateDataVersion; + } + /** * This will create the db version table if it does not already exist * @return @@ -77,7 +93,22 @@ private boolean insureDbVersionTable() { } } - + + /** + * This will create the data version table if it does not already exist + */ + @VisibleForTesting + boolean insureDataVersionTable() { + + try { + currentDataVersion(); + return true; + } catch (Exception e) { + return createDataVersionTable(); + } + + } + /** * Runs with a separate DB connection * @return @@ -91,7 +122,21 @@ private int currentDbVersion() { throw new DotRuntimeException(e); } } - + + /** + * Returns the current data version. Runs with a separate DB connection + */ + @VisibleForTesting + int currentDataVersion() { + try (Connection conn = DbConnectionFactory.getDataSource().getConnection()) { + DotConnect db = new DotConnect().setSQL(SELECT_DATA_VERSION); + return db.loadInt("test"); + + } catch (Exception e) { + throw new DotRuntimeException(e); + } + } + /** * Runs with a separate DB connection * @return @@ -109,10 +154,25 @@ private boolean createDbVersionTable() { } } - - + + /** + * Creates the data version table. Runs with a separate DB connection + */ + private boolean createDataVersionTable() { + + try (Connection conn = DbConnectionFactory.getDataSource().getConnection()) { + new DotConnect().setSQL(createDataVersionTableSQL()).loadResult(conn); + new DotConnect().setSQL(INSERT_DATA_VERSION).addParam(0).addParam(new Date()).loadResult(conn); + return true; + + } catch (Exception e) { + Logger.debug(this.getClass(), e.getMessage(),e); + throw new DotRuntimeException(e); + } + } + public void executeStartUpTasks() throws DotDataException { - + Logger.debug(this.getClass(), "Running Startup Tasks"); @@ -167,9 +227,9 @@ String getTaskId(final String taskName){ } return "-1"; } - - - public void executeUpgrades() throws DotDataException { + + + public void executeSchemaUpgrades() throws DotDataException { Logger.info(this, "---"); Logger.info(this, ""); @@ -227,7 +287,71 @@ public void executeUpgrades() throws DotDataException { } - + + /** + * Runs the data upgrade tasks, those that are not related to the database schema upgrade tasks, + * data tasks are mostly tasks to solve data issues using our existing APIs. + *
+ * The list of data upgrade tasks is obtained from the
+ * {@link TaskLocatorUtil#getStartupRunOnceDataTaskClasses()}
+ *
+ * @throws DotDataException
+ */
+ public void executeDataUpgrades() throws DotDataException {
+
+ Logger.info(this, "---");
+ Logger.info(this, "");
+ Logger.info(this, "Running Data Upgrade Tasks");
+ Logger.info(this, "Database data version: " + Config.DATA_VERSION);
+
+ String name;
+
+ for (Class c : TaskLocatorUtil.getStartupRunOnceDataTaskClasses()) {
+
+ name = c.getCanonicalName();
+ name = name.substring(name.lastIndexOf(".") + 1);
+ String id = getTaskId(name);
+
+ try {
+ int taskId = Integer.parseInt(id);
+ if (StartupTask.class.isAssignableFrom(c) && taskId > Config.DATA_VERSION) {
+ StartupTask task;
+ try {
+ task = (StartupTask) c.newInstance();
+ } catch (Exception e) {
+ throw new DotRuntimeException(e.getMessage(), e);
+ }
+
+ if (!firstTimeStart && task.forceRun()) {
+ HibernateUtil.startTransaction();
+ Logger.info(this, "Running Data Upgrade Tasks: " + name);
+ task.executeUpgrade();
+ }
+
+ new DotConnect()
+ .setSQL(INSERT_DATA_VERSION)
+ .addParam(taskId)
+ .addParam(new Date())
+ .loadResult();
+ Logger.info(this, "Data upgraded to version: " + taskId);
+ HibernateUtil.closeAndCommitTransaction();
+ Config.DATA_VERSION = taskId;
+ }
+ } catch (Exception e) {
+ HibernateUtil.rollbackTransaction();
+ if (Config.getBooleanProperty("SYSTEM_EXIT_ON_STARTUP_FAILURE", true)) {
+ Logger.error(this, "FATAL: " + e.getMessage(), e);
+ System.exit(1);
+ }
+ } finally {
+ HibernateUtil.closeAndCommitTransaction();
+ }
+
+ }
+
+ Logger.info(this, "Finishing data upgrade tasks.");
+ }
+
/**
* This will execute all the UT that were backported to the LTS version.
* Will be run everytime the server gets restarted just like the Startup Tasks.
@@ -247,7 +371,7 @@ public void executeBackportedTasks() throws DotDataException {
name = c.getCanonicalName();
name = name.substring(name.lastIndexOf(".") + 1);
String id = getTaskId(name);
- int taskId = Integer.parseInt(id);
+ int taskId = Integer.parseInt(id);
if (StartupTask.class.isAssignableFrom(c) && taskId > Config.DB_VERSION) {
StartupTask task = (StartupTask) c.newInstance();
if (task.forceRun()) {
diff --git a/dotCMS/src/main/java/com/dotmarketing/util/Config.java b/dotCMS/src/main/java/com/dotmarketing/util/Config.java
index 5eb5a75ac7f9..fadcf05c9fd5 100644
--- a/dotCMS/src/main/java/com/dotmarketing/util/Config.java
+++ b/dotCMS/src/main/java/com/dotmarketing/util/Config.java
@@ -64,6 +64,7 @@ public class Config {
public static final String DOTCMS_USEWATCHERMODE = "dotcms.usewatchermode";
public static final String USE_CONFIG_TEST_OVERRIDE_TRACKER = "USE_CONFIG_TEST_OVERRIDE_TRACKER";
public static int DB_VERSION = 0;
+ public static int DATA_VERSION = 0;
//Object Config properties n
public static javax.servlet.ServletContext CONTEXT = null;
diff --git a/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java b/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java
index 7bdbd75dc2e5..0df5fd2592c6 100644
--- a/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java
+++ b/dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java
@@ -322,7 +322,7 @@ public static List
+ * The number after the "Task" word and before the description represents
+ * the system data version number in the {@code DATA_VERSION} table. A successfully
+ * executed upgrade task will add a new record in such a table with the
+ * number in the class name. This allows dotCMS to keep track of the tasks
+ * that have been run.
+ *
+ * @return The list of Run-Once Data Tasks.
+ */
+ public static List