[1139735] HA upgrade issues with storage cluster schema change

First check-in for the necessary revisions. What makes a storage cluster schema change difficult is that it requires the entire storage cluster to be upgraded to the new bits, and to be running, in order to apply the changes. This runs contrary to our typical upgrade approach which is to shut everything down and upgrade one install at a time (an install being a server and/or storage node). The basic approach is to now track the install version for each installed component, and only when evrything has been upgraded, then do we allow the storage cluster start and then perform the schema update as a separate upgrade step. - Add version stamp on rhq_server and rhq_storage_node - Add db upgrade logic to add fields and init them to the current version - Add Support to stamp the version on install and upgrades - includes updates to jboss modules, rhqctl and installers - includes ability to contact RDB from storage installer - Add --storage-schema and --list-versions coptions to rhqctl upgrade - Stop server startup unless topology is set to homogenous version, to prevent running an environment that is not completely upgraded. -
rhq-project · Oct 3, 2014 · 08d41ec · 08d41ec
1 parent 29368f3
commit 08d41ec
Show file tree

Hide file tree

Showing 21 changed files with 853 additions and 131 deletions.
diff --git a/modules/common/cassandra-installer/pom.xml b/modules/common/cassandra-installer/pom.xml
@@ -17,6 +17,12 @@
   </properties>
 
   <dependencies>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>rhq-core-dbutils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>rhq-cassandra-ccm-core</artifactId>
@@ -95,6 +101,15 @@
                   <groupId>${project.groupId}</groupId>
                   <artifactId>rhq-cassandra-ccm-core</artifactId>
                 </artifactItem>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
+                  <artifactId>rhq-core-dbutils</artifactId>
+                  <version>${project.version}</version>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>i18nlog</groupId>
+                  <artifactId>i18nlog</artifactId>
+                </artifactItem>
                 <artifactItem>
                   <groupId>${project.groupId}</groupId>
                   <artifactId>rhq-core-util</artifactId>

diff --git a/.../common/cassandra-installer/src/main/java/org/rhq/storage/installer/StorageInstaller.java b/.../common/cassandra-installer/src/main/java/org/rhq/storage/installer/StorageInstaller.java
@@ -36,6 +36,9 @@
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.UnknownHostException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -71,10 +74,14 @@
 import org.rhq.cassandra.DeploymentOptions;
 import org.rhq.cassandra.DeploymentOptionsFactory;
 import org.rhq.cassandra.util.ConfigEditor;
+import org.rhq.core.db.DatabaseType;
+import org.rhq.core.db.DatabaseTypeFactory;
+import org.rhq.core.db.DbUtil;
 import org.rhq.core.util.PropertiesFileUpdate;
 import org.rhq.core.util.StringUtil;
 import org.rhq.core.util.exception.ThrowableUtil;
 import org.rhq.core.util.file.FileUtil;
+import org.rhq.core.util.obfuscation.PicketBoxObfuscator;
 import org.rhq.core.util.stream.StreamUtil;
 
 /**
@@ -118,7 +125,7 @@ public class StorageInstaller {
 
     static final String DEFAULT_SAVED_CACHES_DIR = "../../../rhq-data/saved_caches";
 
-    private final Log log = LogFactory.getLog(StorageInstaller.class);
+    static private final Log log = LogFactory.getLog(StorageInstaller.class);
 
     private Options options;
 
@@ -228,10 +235,13 @@ public int run(CommandLine cmdLine) throws Exception {
         }
 
         InstallerInfo installerInfo = null;
+        boolean isUpgrade = cmdLine.hasOption("upgrade");
+        File upgradeFromDir = null;
 
         try {
-            if (cmdLine.hasOption("upgrade")) {
-                installerInfo = upgrade(new File(cmdLine.getOptionValue("upgrade", "")));
+            if (isUpgrade) {
+                upgradeFromDir = new File(cmdLine.getOptionValue("upgrade", ""));
+                installerInfo = upgrade(upgradeFromDir);
             } else {
                 installerInfo = install(cmdLine);
             }
@@ -255,6 +265,21 @@ public int run(CommandLine cmdLine) throws Exception {
 
         serverPropertiesUpdater.update(properties);
 
+        Properties dbProperties = null;
+        if (isUpgrade) {
+            File oldServerPropsFile = new File(upgradeFromDir, "bin/rhq-server.properties");
+            dbProperties = new Properties();
+            FileInputStream oldServerPropsFileInputStream = new FileInputStream(oldServerPropsFile);
+            try {
+                dbProperties.load(oldServerPropsFileInputStream);
+            } finally {
+                oldServerPropsFileInputStream.close();
+            }
+        } else {
+            dbProperties = serverPropertiesUpdater.loadExistingProperties();
+        }
+        stampStorageNodeVersion(dbProperties);
+
         // start node (and install windows service) if necessary
         File binDir = null;
         if (isWindows()) {
@@ -364,7 +389,8 @@ private InstallerInfo install(CommandLine cmdLine) throws StorageInstallerExcept
             String seeds = cmdLine.getOptionValue(StorageProperty.SEEDS.property(), installerInfo.hostname);
             deploymentOptions.setSeeds(seeds);
 
-            String commitlogDir = cmdLine.getOptionValue(StorageProperty.COMMITLOG.property(), getDefaultCommitLogDir());
+            String commitlogDir = cmdLine
+                .getOptionValue(StorageProperty.COMMITLOG.property(), getDefaultCommitLogDir());
             String dataDir = cmdLine.getOptionValue(StorageProperty.DATA.property(), getDefaultDataDir());
             String savedCachesDir = cmdLine.getOptionValue(StorageProperty.SAVED_CACHES.property(),
                 getDefaultSavedCachesDir());
@@ -373,7 +399,6 @@ private InstallerInfo install(CommandLine cmdLine) throws StorageInstallerExcept
             File dataDirFile = new File(dataDir);
             File savedCachesDirFile = new File(savedCachesDir);
 
-
             boolean verifyDataDirsEmpty = Boolean.valueOf(cmdLine.getOptionValue(
                 StorageProperty.VERIFY_DATA_DIRS_EMPTY.property(), "true"));
             if (verifyDataDirsEmpty) {
@@ -470,13 +495,16 @@ private InstallerInfo install(CommandLine cmdLine) throws StorageInstallerExcept
 
     private void verifyPortStatus(CommandLine cmdLine, InstallerInfo installerInfo) throws StorageInstallerException {
         installerInfo.jmxPort = getPort(cmdLine, StorageProperty.JMX_PORT.property(), defaultJmxPort);
-        isPortBound(installerInfo.hostname, installerInfo.jmxPort, StorageProperty.JMX_PORT.property(), STATUS_JMX_PORT_CONFLICT);
+        isPortBound(installerInfo.hostname, installerInfo.jmxPort, StorageProperty.JMX_PORT.property(),
+            STATUS_JMX_PORT_CONFLICT);
 
         installerInfo.cqlPort = getPort(cmdLine, StorageProperty.CQL_PORT.property(), defaultCqlPort);
-        isPortBound(installerInfo.hostname, installerInfo.cqlPort, StorageProperty.CQL_PORT.property(), STATUS_CQL_PORT_CONFLICT);
+        isPortBound(installerInfo.hostname, installerInfo.cqlPort, StorageProperty.CQL_PORT.property(),
+            STATUS_CQL_PORT_CONFLICT);
 
         installerInfo.gossipPort = getPort(cmdLine, StorageProperty.GOSSIP_PORT.property(), defaultGossipPort);
-        isPortBound(installerInfo.hostname, installerInfo.gossipPort, StorageProperty.GOSSIP_PORT.property(), STATUS_GOSSIP_PORT_CONFLICT);
+        isPortBound(installerInfo.hostname, installerInfo.gossipPort, StorageProperty.GOSSIP_PORT.property(),
+            STATUS_GOSSIP_PORT_CONFLICT);
     }
 
     /**
@@ -630,7 +658,8 @@ private void checkPerms(Option option, String path, List<String> errors) {
 
             if (dir.exists()) {
                 if (dir.isFile()) {
-                    errors.add(path + " is not a directory. Use the --" + option.getLongOpt() + " to change this value.");
+                    errors.add(path + " is not a directory. Use the --" + option.getLongOpt()
+                        + " to change this value.");
                 }
             } else {
                 File parentDir = dir.getParentFile();
@@ -639,25 +668,31 @@ private void checkPerms(Option option, String path, List<String> errors) {
                 }
 
                 if (!parentDir.canWrite()) {
-                    errors.add("The user running this installer does not appear to have write permissions to "
-                        + parentDir + ". Either make sure that the user running the storage node has write permissions or use the --"
-                        + option.getLongOpt() + " to change this value.");
+                    errors
+                        .add("The user running this installer does not appear to have write permissions to "
+                            + parentDir
+                            + ". Either make sure that the user running the storage node has write permissions or use the --"
+                            + option.getLongOpt() + " to change this value.");
                 }
             }
         } catch (Exception e) {
-            errors.add("The request path cannot be constructed (path: " + path + "). "
-                + "Please use a valid and also make sure the user running the storage node has write permissions for the path.");
+            errors
+                .add("The request path cannot be constructed (path: "
+                    + path
+                    + "). "
+                    + "Please use a valid and also make sure the user running the storage node has write permissions for the path.");
         }
     }
 
-    private void isPortBound(String address, int port, String portName, int potentialErrorCode) throws StorageInstallerException {
+    private void isPortBound(String address, int port, String portName, int potentialErrorCode)
+        throws StorageInstallerException {
         ServerSocket serverSocket = null;
         try {
             serverSocket = new ServerSocket();
             serverSocket.bind(new InetSocketAddress(address, port));
         } catch (BindException e) {
-            throw new StorageInstallerException("The " + portName + " (" + address + ":" + port + ") is already in use. "
-                    + "Installation cannot proceed.", potentialErrorCode);
+            throw new StorageInstallerException("The " + portName + " (" + address + ":" + port
+                + ") is already in use. " + "Installation cannot proceed.", potentialErrorCode);
         } catch (IOException e) {
             // We only log a warning here and let the installation proceed in case the
             // exception is something that can be ignored.
@@ -681,7 +716,7 @@ private PropertiesFileUpdate getServerProperties() {
 
         File file = new File(sysprop);
         if (!(file.exists() && file.isFile())) {
-            throw new RuntimeException("System property [" + sysprop + "] points to in invalid file.");
+            throw new RuntimeException("System property [" + sysprop + "] points to an invalid file.");
         }
 
         return new PropertiesFileUpdate(file.getAbsolutePath());
@@ -953,6 +988,63 @@ private String getDefaultSavedCachesDir() {
         return DEFAULT_SAVED_CACHES_DIR;
     }
 
+    private static boolean stampStorageNodeVersion(Properties serverProperties) throws Exception {
+        final String dbUrl = serverProperties.getProperty("rhq.server.database.connection-url");
+        final String dbUsername = serverProperties.getProperty("rhq.server.database.user-name");
+        String obfuscatedDbPassword = serverProperties.getProperty("rhq.server.database.password");
+        String clearTextDbPassword = PicketBoxObfuscator.decode(obfuscatedDbPassword);
+
+        boolean result = updateStorageNodeVersion(dbUrl, dbUsername, clearTextDbPassword,
+            serverProperties.getProperty("rhq.storage.nodes"));
+
+        return result;
+    }
+
+    /**
+     * Update server version stamp with the install version.
+     *
+     * @param connectionUrl
+     * @param username
+     * @param password
+     * @param storageNodeAddress
+     * @return true if updated, false otherwise
+     *
+     * @throws Exception if failed to communicate with the database
+     */
+    private static boolean updateStorageNodeVersion(String connectionUrl, String username, String password,
+        String storageNodeAddress) throws Exception {
+        DatabaseType db = null;
+        Connection conn = null;
+        PreparedStatement stm = null;
+        boolean result = false;
+        String version = StorageInstaller.class.getPackage().getImplementationVersion();
+
+        try {
+            conn = DbUtil.getConnection(connectionUrl, username, password);
+            db = DatabaseTypeFactory.getDatabaseType(conn);
+
+            if (db.checkColumnExists(conn, "rhq_storage_node", "version")) {
+
+                stm = conn.prepareStatement("UPDATE rhq_storage_node SET version = ? WHERE address = ?");
+                stm.setString(1, version);
+                stm.setString(2, storageNodeAddress);
+                result = (1 == stm.executeUpdate());
+            }
+        } catch (IllegalStateException e) {
+            log.info("Unable to update storage node [" + storageNodeAddress + "] to version [" + version
+                + "], column does not exist.");
+        } catch (SQLException e) {
+            log.info("Unable to update storage node [" + storageNodeAddress + "] to version [" + version + "] "
+                + e.getMessage());
+        } finally {
+            if (null != db) {
+                db.closeJDBCObjects(conn, stm, null);
+            }
+        }
+
+        return result;
+    }
+
     public void printUsage() {
         HelpFormatter helpFormatter = new HelpFormatter();
         String syntax = "rhq-storage-installer.sh|bat [options]";

diff --git a/modules/common/cassandra-installer/src/main/resources/module/main/module.xml b/modules/common/cassandra-installer/src/main/resources/module/main/module.xml
@@ -6,8 +6,10 @@
   <resources>
     <resource-root path="${project.build.finalName}.jar"/>
     <resource-root path="rhq-cassandra-ccm-core-${project.version}.jar"/>
+    <resource-root path="rhq-core-dbutils-${project.version}.jar"/>
     <resource-root path="rhq-core-util-${project.version}.jar"/>
     <resource-root path="rhq-cassandra-util-${project.version}.jar"/>
+    <resource-root path="i18nlog-${i18nlog.version}.jar"/>
     <resource-root path="snakeyaml-${cassandra.snakeyaml.version}.jar"/>
   </resources>
 
@@ -21,5 +23,8 @@
     <module name="sun.jdk"/>
     <module name="org.jboss.logmanager" services="import"/>
     <module name="org.jboss.logging"/>
+    <module name="org.picketbox"/>
+    <module name="org.rhq.postgres"/>
+    <module name="org.rhq.oracle" optional="true" />
   </dependencies>
 </module>
diff --git a/modules/common/cassandra-schema/src/main/java/org/rhq/cassandra/schema/VersionManager.java b/modules/common/cassandra-schema/src/main/java/org/rhq/cassandra/schema/VersionManager.java
@@ -269,15 +269,15 @@ public void checkCompatibility() throws Exception {
             int requiredSchemaVersion = folder.getLatestVersion();
 
             if (installedSchemaVersion < requiredSchemaVersion) {
-                log.error("Storage cluster schema version:" + installedSchemaVersion + ". Required schema version: "
+                log.warn("Storage cluster schema version:" + installedSchemaVersion + ". Required schema version: "
                     + requiredSchemaVersion + ". Please update storage cluster schema version.");
                 throw new InstalledSchemaTooOldException();
             }
 
             if (installedSchemaVersion > requiredSchemaVersion) {
                 log.error("Storage cluster schema version:" + installedSchemaVersion + ". Required schema version: "
                     + requiredSchemaVersion
-                    + ". Storage clutser has been updated beyond the capability of the current server installation.");
+                    + ". Storage cluster has been updated beyond the capability of the current server installation.");
                 throw new InstalledSchemaTooAdvancedException();
             }
         } catch (NoHostAvailableException e1) {

diff --git a/modules/core/dbutils/pom.xml b/modules/core/dbutils/pom.xml
@@ -17,7 +17,7 @@
     <description>Database schema setup, upgrade and other utilities</description>
 
     <properties>
-        <db.schema.version>2.162</db.schema.version>
+        <db.schema.version>2.163</db.schema.version>
         <rhq.ds.type-mapping>${rhq.test.ds.type-mapping}</rhq.ds.type-mapping>
         <rhq.ds.server-name>${rhq.test.ds.server-name}</rhq.ds.server-name>
         <rhq.ds.db-name>${rhq.test.ds.db-name}</rhq.ds.db-name>

diff --git a/modules/core/dbutils/src/main/java/org/rhq/core/db/upgrade/VersionStampUpgradeTask.java b/modules/core/dbutils/src/main/java/org/rhq/core/db/upgrade/VersionStampUpgradeTask.java
@@ -0,0 +1,32 @@
+package org.rhq.core.db.upgrade;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+
+import mazz.i18n.Logger;
+
+import org.rhq.core.db.DatabaseType;
+import org.rhq.core.db.DbUtilsI18NFactory;
+import org.rhq.core.db.DbUtilsI18NResourceKeys;
+
+/**
+ * Updates the StorageNode and Server version fields to an initial value of the current version.
+ *
+ * @author Jay Shaughnessy
+ */
+public class VersionStampUpgradeTask implements DatabaseUpgradeTask {
+
+    private final Logger log = DbUtilsI18NFactory.getLogger(VersionStampUpgradeTask.class);
+
+    @Override
+    public void execute(DatabaseType databaseType, Connection connection) throws SQLException {
+        String version = this.getClass().getPackage().getImplementationVersion();
+        String update = "UPDATE rhq_server SET version = '" + version + "'";
+        log.debug(DbUtilsI18NResourceKeys.EXECUTING_SQL, update);
+        databaseType.executeSql(connection, update);
+
+        update = "UPDATE rhq_storage_node SET version = '" + version + "'";
+        log.debug(DbUtilsI18NResourceKeys.EXECUTING_SQL, update);
+        databaseType.executeSql(connection, update);
+    }
+}
diff --git a/modules/core/dbutils/src/main/scripts/dbsetup/cluster-schema.xml b/modules/core/dbutils/src/main/scripts/dbsetup/cluster-schema.xml
@@ -19,7 +19,8 @@
         <column name="COMPUTE_POWER" required="true" type="INTEGER"/>        
         <column name="OPERATION_MODE" required="true" size="32" type="VARCHAR2"/>
         <column name="STATUS" type="INTEGER" required="false" default="0" />
-
+        <column name="VERSION" required="true" size="255" type="VARCHAR2" />
+
         <!--  This index is for constraint, not performance -->    
         <index name="RHQ_SERVER_NAME_UNIQUE" unique="true">
             <field ref="NAME"/>

diff --git a/modules/core/dbutils/src/main/scripts/dbsetup/measurement-schema.xml b/modules/core/dbutils/src/main/scripts/dbsetup/measurement-schema.xml
@@ -457,6 +457,7 @@
     <column name="MAINTENANCE_PENDING" required="true" type="BOOLEAN" default="false"/>
     <column name="RESOURCE_ID" required="false" type="INTEGER" references="RHQ_RESOURCE(ID)" />
     <column name="RESOURCE_OP_HIST_ID" required="false" type="INTEGER" references="RHQ_OPERATION_HISTORY(ID)"/>
+    <column name="VERSION" required="true" size="255" type="VARCHAR2" />
 
     <!-- This index is for constraint, not performance -->
     <index name="RHQ_STORAGE_NODE_UNIQUE" unique="true">

diff --git a/modules/core/dbutils/src/main/scripts/dbupgrade/db-upgrade.xml b/modules/core/dbutils/src/main/scripts/dbupgrade/db-upgrade.xml
@@ -2568,6 +2568,15 @@
             <schemaSpec version="2.162">
               <schema-addColumn table="RHQ_BUNDLE_DEPLOYMENT" column="DISCOVERY_DELAY" columnType="INTEGER" />
             </schemaSpec>
+
+            <schemaSpec version="2.163">
+              <schema-addColumn table="RHQ_SERVER"       column="VERSION" columnType="VARCHAR2" precision="255" />
+              <schema-addColumn table="RHQ_STORAGE_NODE" column="VERSION" columnType="VARCHAR2" precision="255" />
+              <schema-javaTask className="VersionStampUpgradeTask" />
+              <schema-alterColumn table="RHQ_SERVER" column="VERSION" nullable="false" />
+              <schema-alterColumn table="RHQ_STORAGE_NODE" column="VERSION" nullable="false" />
+            </schemaSpec>
+
         </dbupgrade>
     </target>
 </project>