BZ 1152154 - wait a few times during startup to ensure the agent has …

…been given a fair chance to register before aborting

(cherry picked from commit 6b97b5c)

modules/enterprise/agent/src/main/java/org/rhq/enterprise/agent/AgentMain.java

@@ -715,6 +715,14 @@ public void start() throws Exception {
                     startManagementServices(); // we start our metric collectors before plugin container so the agent plugin can work
                     boolean mustRegister = prepareStartupWorkRequiringServer();
                     boolean keepWaitingForServer;
+                    int serverUpWaitCounter;
+                    try {
+                        // we really should not need to have to customize this, but make a backdoor setting, just in case
+                        serverUpWaitCounter = Integer.parseInt(System.getProperty(
+                            "rhq.agent.startup-registration-waits", "5"));
+                    } catch (Exception e) {
+                        serverUpWaitCounter = 5;
+                    }
                     do {
                         boolean aServerIsKnownToBeUp = waitForServer(m_configuration.getWaitForServerAtStartupMsecs());
                         boolean agentIsRegistered = isRegistered();
@@ -729,8 +737,15 @@ public void start() throws Exception {
                         } else if (mustRegister && !agentIsRegistered) {
                             // If we got here, we know a server is up and the agent needs to be registered, but it isn't registered.
                             // This usually means an unrecoverable registration error occurred, so abort.
-                            throw new AgentRegistrationException(
-                                MSG.getMsg(AgentI18NResourceKeys.AGENT_CANNOT_REGISTER));
+                            // But there have been cases were there is a race condition and the server just isn't ready yet
+                            // so let's at least retry a few times before giving up (BZ 1152154)
+                            if (--serverUpWaitCounter < 0) {
+                                throw new AgentRegistrationException(
+                                    MSG.getMsg(AgentI18NResourceKeys.AGENT_CANNOT_REGISTER));
+                            } else {
+                                keepWaitingForServer = true;
+                                LOG.warn(AgentI18NResourceKeys.STARTUP_REGISTRATION_FAILED_RETRY, serverUpWaitCounter);
+                            }
                         } else {
                             keepWaitingForServer = false;
                         }

modules/enterprise/agent/src/main/java/org/rhq/enterprise/agent/i18n/AgentI18NResourceKeys.java

@@ -1626,6 +1626,9 @@ public interface AgentI18NResourceKeys {
     @I18NMessage("The name of this agent was not predefined so it was auto-generated. The agent name is now [{0}]")
     String AGENT_NAME_AUTO_GENERATED = "AgentMain.agent-name-auto-generated";
 
+    @I18NMessage("A server appears to be up, but the agent is not yet registered. Will wait again. wait counter=[{0}]")
+    String STARTUP_REGISTRATION_FAILED_RETRY = "AgentMain.startup-registration-failed-retry";
+
     @I18NMessage("Agent is not starting the plugin container at startup, as per its configuration")
     String NOT_STARTING_PLUGIN_CONTAINER_AT_STARTUP = "AgentMain.not-starting-pc-at-startup";