[1139735, 1157480] More upgrade use cases that didn't work.

[1139735] An upgrade with a number of storage nodes > than our replication
factor (>= 4) could fail because our check for the storage cluster
version can fail if the auth info or version info are not replicated
to the currently running SNs.  To avoid this, for ugrades we now push
all storage cluster interaction to the post-upgrade step (i.e.
rhqctl --storage-schema).  This includes the schema creation for a new,
remote storage cluster.

[1157480] An upgrade of standalone storage node can fail because those
installs may not have set proper DB props in rhq-server.properties.  It
wasn't required in the past but it is now.  In the code we now make sure
the properties are copied forward on upgrade. But there is also doco needed
here because prior to upgrade from an earlier version, using standalone
SNs, the old rhq-server.properties files will need to be updated.

Also, update the --list-versions report to better reflect that it may need
all SNs to be running to perform the storage schema version check.

(cherry picked from commit 1b45c2a)
Signed-off-by: Jay Shaughnessy <jshaughn@redhat.com>

modules/common/cassandra-installer/src/main/java/org/rhq/storage/installer/StorageInstaller.java

@@ -221,8 +221,7 @@ public StorageInstaller() {
             .addOption(commitLogOption).addOption(dataDirOption).addOption(savedCachesDirOption)
             .addOption(cqlPortOption).addOption(gossipPortOption).addOption(basedirOption).addOption(heapSizeOption)
             .addOption(heapNewSizeOption).addOption(noVersionStampOption).addOption(stackSizeOption)
-            .addOption(upgradeOption).addOption(undoOption)
-            .addOption(verifyDataDirsEmptyOption);
+            .addOption(upgradeOption).addOption(undoOption).addOption(verifyDataDirsEmptyOption);
     }
 
     public int run(CommandLine cmdLine) throws Exception {
@@ -266,24 +265,35 @@ public int run(CommandLine cmdLine) throws Exception {
         properties.setProperty(StorageProperty.CQL_PORT.property(), Integer.toString(installerInfo.cqlPort));
         properties.setProperty(StorageProperty.GOSSIP_PORT.property(), Integer.toString(installerInfo.gossipPort));
 
-        serverPropertiesUpdater.update(properties);
-
-        Properties dbProperties;
-        if (isUpgrade && !noStamp) {
+        // carry forward the required db props from the legacy install. We need these to contact the
+        // DB to do a version stamp.
+        if (isUpgrade) {
             File oldServerPropsFile = new File(fromDir, "bin/rhq-server.properties");
-            dbProperties = new Properties();
+            Properties oldProperties = new Properties();
             FileInputStream oldServerPropsFileInputStream = new FileInputStream(oldServerPropsFile);
             try {
-                dbProperties.load(oldServerPropsFileInputStream);
+                oldProperties.load(oldServerPropsFileInputStream);
+                properties.setProperty("rhq.server.database.connection-url",
+                    oldProperties.getProperty("rhq.server.database.connection-url"));
+                properties.setProperty("rhq.server.database.user-name",
+                    oldProperties.getProperty("rhq.server.database.user-name"));
+                properties.setProperty("rhq.server.database.password",
+                    oldProperties.getProperty("rhq.server.database.password"));
+
             } finally {
                 oldServerPropsFileInputStream.close();
             }
+        }
 
+        // update the properties file
+        serverPropertiesUpdater.update(properties);
+
+        Properties dbProperties = serverPropertiesUpdater.loadExistingProperties();
+
+        // when upgrading, mark the upgrade by stamping the new version
+        if (isUpgrade && !noStamp) {
             String version = StorageInstaller.class.getPackage().getImplementationVersion();
             stampStorageNodeVersion(dbProperties, installerInfo.hostname, version);
-
-        } else {
-            dbProperties = serverPropertiesUpdater.loadExistingProperties();
         }
 
         // start node (and install windows service) if necessary
@@ -1083,7 +1093,9 @@ private static void updateStorageNodeVersion(String connectionUrl, String userna
             }
         } catch (Exception e) {
             throw new RuntimeException("Unable to update Storage Node [" + storageNodeAddress + "] to version ["
-                + version + "]", e);
+                + version
+                + "].  Make sure the rhq-server.properties file has the correct database property settings! Cause: "
+                + e.getMessage());
         } finally {
             JDBCUtil.safeClose(connection);
         }

modules/enterprise/server/installer/src/main/java/org/rhq/enterprise/server/installer/Installer.java

@@ -50,7 +50,7 @@ public class Installer {
     private InstallerConfiguration installerConfig;
 
     private enum WhatToDo {
-        DISPLAY_USAGE, DO_NOTHING, TEST, SETUPDB, LIST_SERVERS, INSTALL, UPDATESTORAGESCHEMA, LIST_VERSIONS
+        DISPLAY_USAGE, DO_NOTHING, TEST, SETUPDB, LIST_SERVERS, INSTALL, UPDATESTORAGESCHEMA, LIST_VERSIONS, UPGRADE
     }
 
     public static void main(String[] args) {
@@ -107,7 +107,7 @@ public void doInstall(String[] args) throws Exception {
                 try {
                     final InstallerService installerService = new InstallerServiceImpl(installerConfig);
                     final HashMap<String, String> serverProperties = installerService.getServerProperties();
-                    installerService.prepareDatabase(serverProperties, null, null);
+                    installerService.prepareDatabase(serverProperties, null, null, false);
                     LOG.info("Database setup is complete.");
                 } catch (Exception e) {
                     LOG.error(ThrowableUtil.getAllMessages(e));
@@ -127,11 +127,12 @@ public void doInstall(String[] args) throws Exception {
                 }
                 continue;
             }
-            case INSTALL: {
+            case INSTALL:
+            case UPGRADE: {
                 try {
                     final InstallerService installerService = new InstallerServiceImpl(installerConfig);
                     final HashMap<String, String> serverProperties = installerService.preInstall();
-                    installerService.install(serverProperties, null, null);
+                    installerService.install(serverProperties, null, null, (WhatToDo.UPGRADE == whatToDo));
                     LOG.info("Installation is complete. The server should be ready shortly.");
                 } catch (AutoInstallDisabledException e) {
                     LOG.error(e.getMessage());
@@ -167,7 +168,9 @@ private void displayUsage() {
             .append("\n");
         usage.append("\t--setupdb, -b: only perform database schema creation or update").append("\n");
         usage.append("\t--updatestorageschema, -u: only perform storage cluster schema update").append("\n");
-        usage.append("\t--encodevalue, -e: prompts for password or value to encode for editing configuration files for agent or server");
+        usage.append("\t--upgrade, -g: this is an upgrade installation (as opposed to a new install)").append("\n");
+        usage
+            .append("\t--encodevalue, -e: prompts for password or value to encode for editing configuration files for agent or server");
         usage.append("\n");
         LOG.info(usage);
     }
@@ -180,6 +183,7 @@ private WhatToDo[] processArguments(String[] args) throws Exception {
             new LongOpt("encodevalue", LongOpt.NO_ARGUMENT, null, 'e'),
             new LongOpt("setupdb", LongOpt.NO_ARGUMENT, null, 'b'),
             new LongOpt("updatestorageschema", LongOpt.NO_ARGUMENT, null, 'u'),
+            new LongOpt("upgrade", LongOpt.NO_ARGUMENT, null, 'g'),
             new LongOpt("listservers", LongOpt.NO_ARGUMENT, null, 'l'),
             new LongOpt("listversions", LongOpt.NO_ARGUMENT, null, 'v'),
             new LongOpt("force", LongOpt.NO_ARGUMENT, null, 'f'), new LongOpt("test", LongOpt.NO_ARGUMENT, null, 't') };
@@ -188,6 +192,7 @@ private WhatToDo[] processArguments(String[] args) throws Exception {
         boolean listservers = false;
         boolean listversions = false;
         boolean setupdb = false;
+        boolean upgrade = false;
         boolean updatestorage = false;
         String valueToEncode = null;
         String associatedProperty = null;
@@ -285,6 +290,11 @@ private WhatToDo[] processArguments(String[] args) throws Exception {
                 break; // don't return, in case we need to allow more args
             }
 
+            case 'g': {
+                upgrade = true;
+                break; // don't return, in case we need to allow more args
+            }
+
             case 'u': {
                 updatestorage = true;
                 break; // don't return, in case we need to allow more args
@@ -339,20 +349,23 @@ private WhatToDo[] processArguments(String[] args) throws Exception {
                 }
 
                 System.out.println("!!! WARNING !!!");
-                System.out.println("Both standalone-full.xml and rhq-server.properties need to be updated if a property from rhq-server.properties is used in standalone-full.xml");
+                System.out
+                    .println("Both standalone-full.xml and rhq-server.properties need to be updated if a property from rhq-server.properties is used in standalone-full.xml");
                 System.out.println("!!! WARNING !!!");
                 System.out.println("     ");
                 System.out.println("Encoded " + prompt + " for rhq-server.properties:");
                 System.out.println("     " + associatedProperty + "=RESTRICTED::" + encodedValue);
                 System.out.println("     ");
-                System.out.println("Encoded " + prompt + " for standalone-full.xml with selected " + prompt + " as default:");
+                System.out.println("Encoded " + prompt + " for standalone-full.xml with selected " + prompt
+                    + " as default:");
                 System.out.println("     ${VAULT::restricted::" + associatedProperty + "::" + encodedValue + "}");
                 System.out.println("     ");
                 System.out.println("Encoded " + prompt + " for standalone-full.xml without default:");
                 System.out.println("     ${VAULT::restricted::" + associatedProperty + ":: }");
                 System.out.println("     ");
                 System.out.println("Encoded " + prompt + " for agent-configuration.xml:");
-                System.out.println("     <entry key=\"" + associatedProperty + "\" value=\"RESTRICTED::" + encodedValue + "\" />");
+                System.out.println("     <entry key=\"" + associatedProperty + "\" value=\"RESTRICTED::" + encodedValue
+                    + "\" />");
                 System.out.println("     ");
             }
 
@@ -382,7 +395,7 @@ private WhatToDo[] processArguments(String[] args) throws Exception {
             return whatToDo.toArray(new WhatToDo[whatToDo.size()]);
         }
 
-        return new WhatToDo[] { WhatToDo.INSTALL };
+        return new WhatToDo[] { (upgrade ? WhatToDo.UPGRADE : WhatToDo.INSTALL) };
     }
 
     private String ask(Console console, String prompt) {

modules/enterprise/server/installer/src/main/java/org/rhq/enterprise/server/installer/InstallerService.java

@@ -100,11 +100,14 @@ public interface InstallerService {
      *                             existing schema. Must be one of the names of the
      *                             {@link ServerInstallUtil.ExistingSchemaOption} enum.
      *                             If in auto-install mode, this value is ignored and can be anything.
+     * @param isUpgrade If true this is an upgrade install (rhqctl upgrade), otherwise just an install (rhqctl install)
+     *
      * @throws AutoInstallDisabledException if the server configuration properties does not have auto-install enabled
      * @throws AlreadyInstalledException if it appears the installer was already run and the server is fully installed
      * @throws Exception some other exception that should disallow the installation from continuing
      */
-    void install(HashMap<String, String> serverProperties, ServerDetails serverDetails, String existingSchemaOption)
+    void install(HashMap<String, String> serverProperties, ServerDetails serverDetails, String existingSchemaOption,
+        boolean isUpgrade)
         throws AutoInstallDisabledException, AlreadyInstalledException, Exception;
 
     /**
@@ -126,15 +129,17 @@ void install(HashMap<String, String> serverProperties, ServerDetails serverDetai
      *                             existing schema. Must be one of the names of the
      *                             {@link ServerInstallUtil.ExistingSchemaOption} enum.
      *                             If in auto-install mode, this value is ignored and can be anything.
+     * @param isUpgrade If true this is an upgrade install (rhqctl upgrade), otherwise just an install (rhqctl install)
+     *
      * @throws Exception failed to successfully prepare the database
      */
     void prepareDatabase(HashMap<String, String> serverProperties, ServerDetails serverDetails,
-        String existingSchemaOption) throws Exception;
+        String existingSchemaOption, boolean isUpgrade) throws Exception;
 
     /**
-     * Update the existing storage cluster schema.  All storage nodes must be up and running the bits associated
-     * with this schema.<b/>
-     * NOTE: if not in auto-install mode, the database password is assumed to be in clear text (i.e. unencoded).
+     * Update an existing storage cluster schema or install a new schema if it does not yet exist.  All storage nodes
+     * must be up and running the bits associated with this schema.<b/>
+     * NOTE: if not in auto-install mode, the database password is assumed to be in clear text (i.e. not encoded).
      * If in auto-install mode, the database password must be encoded already.
      *
      * @param serverProperties the server's settings to use. These will be persisted to