Improved liquibase client and Timestamp scripts to be run only when r…

…equired Signed-off-by: coduz <alberto.codutti@eurotech.com>
eclipse · Mar 20, 2020 · 47c94a4 · 47c94a4
1 parent 4d0ac81
commit 47c94a4
Show file tree

Hide file tree

Showing 42 changed files with 236 additions and 110 deletions.
diff --git a/broker-core/src/main/java/org/eclipse/kapua/broker/core/KapuaDatabaseCheckUpdate.java b/broker-core/src/main/java/org/eclipse/kapua/broker/core/KapuaDatabaseCheckUpdate.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2018 Eurotech and/or its affiliates and others
+ * Copyright (c) 2018, 2020 Eurotech and/or its affiliates and others
  *
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
@@ -11,43 +11,39 @@
  *******************************************************************************/
 package org.eclipse.kapua.broker.core;
 
-import java.util.Optional;
-
+import com.google.common.base.MoreObjects;
 import org.eclipse.kapua.commons.jpa.JdbcConnectionUrlResolvers;
+import org.eclipse.kapua.commons.liquibase.KapuaLiquibaseClient;
 import org.eclipse.kapua.commons.setting.system.SystemSetting;
 import org.eclipse.kapua.commons.setting.system.SystemSettingKey;
-import org.eclipse.kapua.commons.liquibase.KapuaLiquibaseClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.MoreObjects;
-
 /**
  * Call Liquibase database schema check and update (if enabled)
- *
  */
 public class KapuaDatabaseCheckUpdate {
 
     private static final Logger logger = LoggerFactory.getLogger(KapuaDatabaseCheckUpdate.class);
 
-    public KapuaDatabaseCheckUpdate() throws Exception {
+    public KapuaDatabaseCheckUpdate() {
         logger.info("Kapua database schema check and update...");
         try {
             SystemSetting config = SystemSetting.getInstance();
-            if(config.getBoolean(SystemSettingKey.DB_SCHEMA_UPDATE, false)) {
+            if (config.getBoolean(SystemSettingKey.DB_SCHEMA_UPDATE, false)) {
                 logger.debug("Starting Liquibase embedded client.");
                 String dbUsername = config.getString(SystemSettingKey.DB_USERNAME);
                 String dbPassword = config.getString(SystemSettingKey.DB_PASSWORD);
                 String schema = MoreObjects.firstNonNull(config.getString(SystemSettingKey.DB_SCHEMA_ENV), config.getString(SystemSettingKey.DB_SCHEMA));
-                new KapuaLiquibaseClient(JdbcConnectionUrlResolvers.resolveJdbcUrl(), dbUsername, dbPassword, Optional.of(schema)).update();
+
+                new KapuaLiquibaseClient(JdbcConnectionUrlResolvers.resolveJdbcUrl(), dbUsername, dbPassword, schema).update();
                 logger.info("Kapua database schema check and update... DONE");
-            }
-            else {
+            } else {
                 logger.info("Kapua database schema check and update... skipping (not enabled by configuration) DONE");
             }
-        } catch (Throwable t) {
-            logger.error("Error in plugin installation.", t);
-            throw new SecurityException(t);
+        } catch (Exception e) {
+            logger.error("Error in plugin installation.", e);
+            throw new SecurityException(e);
         }
     }
 

diff --git a/commons/src/main/java/org/eclipse/kapua/commons/liquibase/KapuaLiquibaseClient.java b/commons/src/main/java/org/eclipse/kapua/commons/liquibase/KapuaLiquibaseClient.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2017, 2018 Red Hat Inc and others.
+ * Copyright (c) 2017, 2020 Red Hat Inc and others.
  *
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
@@ -12,6 +12,7 @@
  *******************************************************************************/
 package org.eclipse.kapua.commons.liquibase;
 
+import com.google.common.base.Strings;
 import liquibase.Liquibase;
 import liquibase.database.Database;
 import liquibase.database.DatabaseFactory;
@@ -21,62 +22,82 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.SystemUtils;
+import org.eclipse.kapua.commons.liquibase.settings.LiquibaseClientSettingKeys;
+import org.eclipse.kapua.commons.liquibase.settings.LiquibaseClientSettings;
+import org.eclipse.kapua.commons.util.SemanticVersion;
 import org.reflections.Reflections;
 import org.reflections.scanners.ResourcesScanner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.FileOutputStream;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.net.URL;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 import java.util.regex.Pattern;
 
 public class KapuaLiquibaseClient {
 
     private static final Logger LOG = LoggerFactory.getLogger(KapuaLiquibaseClient.class);
 
+    private static final SemanticVersion LIQUIBASE_TIMESTAMP_FIX_VERSION = new SemanticVersion("3.3.3"); // https://liquibase.jira.com/browse/CORE-1958
+
+    private static final LiquibaseClientSettings LIQUIBASE_CLIENT_SETTINGS = LiquibaseClientSettings.getInstance();
+
     private final String jdbcUrl;
     private final String username;
     private final String password;
-    private final Optional<String> schema;
+    private final String schema;
+    private final boolean runTimestampsFix;
 
-    public KapuaLiquibaseClient(String jdbcUrl, String username, String password, Optional<String> schema) {
+    public KapuaLiquibaseClient(String jdbcUrl, String username, String password, String schema) {
         this.jdbcUrl = jdbcUrl;
         this.username = username;
         this.password = password;
         this.schema = schema;
+
+        // Check wether or not fix the timestamp based on Liquibase version
+        boolean forceTimestampFix = LIQUIBASE_CLIENT_SETTINGS.getBoolean(LiquibaseClientSettingKeys.FORCE_TIMESTAMPS_FIX);
+        String currentLiquibaseVersionString = LIQUIBASE_CLIENT_SETTINGS.getString(LiquibaseClientSettingKeys.LIQUIBASE_VERSION);
+        SemanticVersion currentLiquibaseVersion = new SemanticVersion(currentLiquibaseVersionString);
+
+        runTimestampsFix = (currentLiquibaseVersion.afterOrMatches(LIQUIBASE_TIMESTAMP_FIX_VERSION) || forceTimestampFix);
     }
 
     public KapuaLiquibaseClient(String jdbcUrl, String username, String password) {
-        this(jdbcUrl, username, password, Optional.empty());
+        this(jdbcUrl, username, password, null);
     }
 
     public void update() {
         try {
             if (Boolean.parseBoolean(System.getProperty("LIQUIBASE_ENABLED", "true")) || Boolean.parseBoolean(System.getenv("LIQUIBASE_ENABLED"))) {
-                LOG.info("Running Liquibase update with schema: " + schema.toString());
+                LOG.info("Running Liquibase update with schema: {}", schema);
                 try (Connection connection = DriverManager.getConnection(jdbcUrl, username, password)) {
-                    loadResourcesStatic(connection, schema);
+                    File changelogDir = loadChangelogs();
+
+                    List<String> contexts = new ArrayList<>();
+                    if (!runTimestampsFix) {
+                        contexts.add("!fixTimestamps");
+                }
+
+                    executeMasters(connection, schema, changelogDir, contexts);
                 }
             }
         } catch (LiquibaseException | SQLException | IOException e) {
-            LOG.error("Error while running Liquibase scripts!", e);
+            LOG.error("Error while running Liquibase scripts: {}", e.getMessage(), e);
             throw new RuntimeException(e);
         }
     }
 
-    protected static synchronized void loadResourcesStatic(Connection connection, Optional<String> schema) throws LiquibaseException, IOException {
-        //
-        // Copy files to temporary directory
+    protected static synchronized File loadChangelogs() throws IOException {
         String tmpDirectory = SystemUtils.getJavaIoTmpDir().getAbsolutePath();
 
         File changelogTempDirectory = new File(tmpDirectory, "kapua-liquibase");
@@ -85,58 +106,62 @@ protected static synchronized void loadResourcesStatic(Connection connection, Op
             FileUtils.deleteDirectory(changelogTempDirectory);
         }
 
-        changelogTempDirectory.mkdirs();
-        LOG.trace("Tmp dir: {}", changelogTempDirectory.getAbsolutePath());
+        boolean createdTmp = changelogTempDirectory.mkdirs();
+        LOG.trace("{} Tmp dir: {}", createdTmp ? "Created" : "Using", changelogTempDirectory.getAbsolutePath());
 
         Reflections reflections = new Reflections("liquibase", new ResourcesScanner());
         Set<String> changeLogs = reflections.getResources(Pattern.compile(".*\\.xml|.*\\.sql"));
         for (String script : changeLogs) {
             URL scriptUrl = KapuaLiquibaseClient.class.getResource("/" + script);
             File changelogFile = new File(changelogTempDirectory, script.replaceFirst("liquibase/", ""));
             if (changelogFile.getParentFile() != null && !changelogFile.getParentFile().exists()) {
-                LOG.trace("Creating parent dir: {}", changelogFile.getParentFile().getAbsolutePath());
-                changelogFile.getParentFile().mkdirs();
+                boolean createdParent = changelogFile.getParentFile().mkdirs();
+                LOG.trace("{} parent dir: {}", createdParent ? "Created" : "Using", changelogFile.getParentFile().getAbsolutePath());
             }
             try (FileOutputStream tmpStream = new FileOutputStream(changelogFile)) {
                 IOUtils.write(IOUtils.toString(scriptUrl), tmpStream);
             }
             LOG.trace("Copied file: {}", changelogFile.getAbsolutePath());
         }
 
+        return changelogTempDirectory;
+    }
+
+    protected static void executeMasters(Connection connection, String schema, File changelogDir, List<String> contexts) throws LiquibaseException {
         //
         // Find and execute all master scripts
         LOG.info("Executing pre master files...");
-        executeMasters(connection, schema, changelogTempDirectory, "-master.pre.xml");
+        executeMasters(connection, schema, changelogDir, "-master.pre.xml", contexts);
         LOG.info("Executing pre master files... DONE!");
 
         LOG.info("Executing master files...");
-        executeMasters(connection, schema, changelogTempDirectory, "-master.xml");
+        executeMasters(connection, schema, changelogDir, "-master.xml", contexts);
         LOG.info("Executing master files... DONE!");
 
         LOG.info("Executing post master files...");
-        executeMasters(connection, schema, changelogTempDirectory, "-master.post.xml");
+        executeMasters(connection, schema, changelogDir, "-master.post.xml", contexts);
         LOG.info("Executing post master files... DONE!");
-
     }
 
-    private static void executeMasters(Connection connection, Optional<String> schema, File changelogTempDirectory, String preMaster) throws LiquibaseException {
-        List<File> masterChangelogs = Arrays.asList(changelogTempDirectory.listFiles((FilenameFilter) (dir, name) -> name.endsWith(preMaster)));
+    protected static void executeMasters(Connection connection, String schema, File changelogTempDirectory, String preMaster, List<String> contexts) throws LiquibaseException {
+        List<File> masterChangelogs = Arrays.asList(changelogTempDirectory.listFiles((dir, name) -> name.endsWith(preMaster)));
 
         LOG.info("\tMaster Liquibase files found: {}", masterChangelogs.size());
 
         LOG.trace("\tSorting master Liquibase files found.");
-        masterChangelogs.sort((f1, f2) -> f1.getAbsolutePath().compareTo(f2.getAbsolutePath()));
+        masterChangelogs.sort(Comparator.comparing(File::getAbsolutePath));
 
+        String ctx = contexts.isEmpty() ? null : String.join(",", contexts);
         for (File masterChangelog : masterChangelogs) {
-            LOG.info("\t\tExcuting liquibase script: {}", masterChangelog.getAbsolutePath());
+            LOG.info("\t\tExecuting liquibase script: {}...", masterChangelog.getAbsolutePath());
             Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(connection));
-            if (schema.isPresent()) {
-                database.setDefaultSchemaName(schema.get());
+            if (!Strings.isNullOrEmpty(schema)) {
+                database.setDefaultSchemaName(schema);
             }
             Liquibase liquibase = new Liquibase(masterChangelog.getAbsolutePath(), new FileSystemResourceAccessor(), database);
-            liquibase.update(null);
+            liquibase.update(ctx);
 
-            LOG.debug("\t\tExcuted liquibase script: {}", masterChangelog.getAbsolutePath());
+            LOG.debug("\t\tExecuting liquibase script: {}... DONE!", masterChangelog.getAbsolutePath());
         }
     }
 

diff --git a/...rc/main/java/org/eclipse/kapua/commons/liquibase/settings/LiquibaseClientSettingKeys.java b/...rc/main/java/org/eclipse/kapua/commons/liquibase/settings/LiquibaseClientSettingKeys.java
@@ -0,0 +1,57 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Eurotech and/or its affiliates and others
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Eurotech - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.kapua.commons.liquibase.settings;
+
+import org.eclipse.kapua.commons.setting.SettingKey;
+
+/**
+ * {@link LiquibaseClientSettings} {@link SettingKey}s
+ *
+ * @since 1.2.0
+ */
+public enum LiquibaseClientSettingKeys implements SettingKey {
+
+    /**
+     * Whether or not force the fix of the SQL timestamps.
+     *
+     * @since 1.2.0
+     */
+    FORCE_TIMESTAMPS_FIX("liquibaseClient.timestamps.fix.force"),
+
+    /**
+     * {@link liquibase.Liquibase} version.
+     *
+     * @since 1.2.0
+     */
+    LIQUIBASE_VERSION("liquibaseClient.liquibase.version");
+
+
+    /**
+     * The {@link String} {@link LiquibaseClientSettingKeys} name.
+     */
+    private final String key;
+
+    /**
+     * Constructor.
+     *
+     * @param key The {@link String} {@link LiquibaseClientSettingKeys} name.
+     * @since 1.2.0
+     */
+    private LiquibaseClientSettingKeys(String key) {
+        this.key = key;
+    }
+
+    @Override
+    public String key() {
+        return key;
+    }
+}
diff --git a/...s/src/main/java/org/eclipse/kapua/commons/liquibase/settings/LiquibaseClientSettings.java b/...s/src/main/java/org/eclipse/kapua/commons/liquibase/settings/LiquibaseClientSettings.java
@@ -0,0 +1,44 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Eurotech and/or its affiliates and others
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Eurotech - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.kapua.commons.liquibase.settings;
+
+import org.eclipse.kapua.commons.setting.AbstractKapuaSetting;
+
+/**
+ * @since 1.2.0
+ */
+public class LiquibaseClientSettings extends AbstractKapuaSetting<LiquibaseClientSettingKeys> {
+
+    private static final String CONFIG_RESOURCE_NAME = "liquibase-client-settings.properties";
+
+    private static final LiquibaseClientSettings INSTANCE = new LiquibaseClientSettings();
+
+    /**
+     * Constructor.
+     *
+     * @since 1.2.0
+     */
+    private LiquibaseClientSettings() {
+        super(CONFIG_RESOURCE_NAME);
+    }
+
+    /**
+     * Gets the {@link LiquibaseClientSettings} singleton instance.
+     *
+     * @return The {@link LiquibaseClientSettings} singleton instance.
+     * @since 1.2.0
+     */
+    public static LiquibaseClientSettings getInstance() {
+        return INSTANCE;
+    }
+
+}
diff --git a/commons/src/main/resources/liquibase-client-settings.properties b/commons/src/main/resources/liquibase-client-settings.properties
@@ -0,0 +1,14 @@
+###############################################################################
+# Copyright (c) 2020 Eurotech and/or its affiliates and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+#
+# Contributors:
+#     Eurotech - initial API and implementation
+#
+###############################################################################
+liquibaseClient.liquibase.version=${liquibase.version}
+liquibaseClient.timestamps.fix.force=false
diff --git a/commons/src/main/resources/liquibase/1.2.0/sys-event-store-timestamp.xml b/commons/src/main/resources/liquibase/1.2.0/sys-event-store-timestamp.xml
@@ -19,7 +19,7 @@
 
     <include relativeToChangelogFile="true" file="../common-properties.xml"/>
 
-    <changeSet id="changelog-event_store-1.2.0-timestamp" author="eurotech">
+    <changeSet id="changelog-event_store-1.2.0-timestamp" author="eurotech" dbms="mysql, mariadb" context="fixTimestamps">
         <modifyDataType tableName="sys_event_store" columnName="created_on" newDataType="timestamp(3) DEFAULT ${now}"/>
         <modifyDataType tableName="sys_event_store" columnName="modified_on" newDataType="timestamp(3) DEFAULT ${now}"/>
         <modifyDataType tableName="sys_event_store" columnName="event_on" newDataType="timestamp(3) DEFAULT ${now}"/>

diff --git a/commons/src/main/resources/liquibase/1.2.0/sys-housekeeper-timestamp.xml b/commons/src/main/resources/liquibase/1.2.0/sys-housekeeper-timestamp.xml
@@ -19,7 +19,7 @@
 
     <include relativeToChangelogFile="true" file="../common-properties.xml"/>
 
-    <changeSet id="changelog-housekeeper-1.2.0-timestamp" author="eurotech">
+    <changeSet id="changelog-housekeeper-1.2.0-timestamp" author="eurotech" dbms="mysql, mariadb" context="fixTimestamps">
         <modifyDataType tableName="sys_housekeeper_run" columnName="last_run_on" newDataType="timestamp(3) NULL"/>
     </changeSet>
 

diff --git a/commons/src/main/resources/liquibase/1.2.0/system_configuration-timestamp.xml b/commons/src/main/resources/liquibase/1.2.0/system_configuration-timestamp.xml
@@ -17,7 +17,7 @@
                       http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.0.xsd"
         logicalFilePath="KapuaDB/changelog-system_configuration-1.2.0.xml">
 
-    <changeSet id="changelog-system_configuration-1.2.0_timestamp" author="eurotech" dbms="mysql, mariadb">
+    <changeSet id="changelog-system_configuration-1.2.0_timestamp" author="eurotech" dbms="mysql, mariadb" context="fixTimestamps">
         <modifyDataType tableName="sys_configuration" columnName="created_on" newDataType="timestamp(3) DEFAULT ${now}"/>
         <modifyDataType tableName="sys_configuration" columnName="modified_on" newDataType="timestamp(3) DEFAULT ${now}"/>
     </changeSet>