[1094540] Agents get permanently backfilled if backend database goes …

…offline for a moment

Don't unset the backfill flag in the ping logic, just make
sure the agent knows the server thinks its down, and let it
resolve the situation by sending a full avail report, which when
processed will unset the backfill flag.

modules/core/client-api/src/main/java/org/rhq/core/clientapi/server/core/PingRequest.java

@@ -24,7 +24,7 @@
 
 /**
  * A simple POJO for requesting actions or data from the ping.
- * 
+ *
  * @author Jay Shaughnessy
  */
 public class PingRequest implements Serializable {
@@ -36,6 +36,7 @@ public class PingRequest implements Serializable {
 
     private boolean replyUpdateAvailability;
     private Long replyServerTimestamp;
+    private boolean replyAgentIsBackfilled;
 
     public PingRequest(String agentName) {
         this(agentName, true, true);
@@ -74,4 +75,13 @@ public Long getReplyServerTimestamp() {
     public void setReplyServerTimestamp(Long replyServerTimestamp) {
         this.replyServerTimestamp = replyServerTimestamp;
     }
+
+    public boolean isReplyAgentIsBackfilled() {
+        return replyAgentIsBackfilled;
+    }
+
+    public void setReplyAgentIsBackfilled(boolean replyAgentIsBackfilled) {
+        this.replyAgentIsBackfilled = replyAgentIsBackfilled;
+    }
+
 }

modules/core/domain/src/main/java/org/rhq/core/domain/resource/Agent.java

@@ -150,7 +150,11 @@
         + "  WHERE id = :agentId "), //
     @NamedQuery(name = Agent.QUERY_UPDATE_LAST_AVAIL_PING, query = "" //
         + " UPDATE Agent a " //
-        + "    SET lastAvailabilityPing = :now, backFilled = FALSE " //
+        + "    SET lastAvailabilityPing = :now " //
+        + "  WHERE name = :agentName AND backfilled = FALSE "), //
+    @NamedQuery(name = Agent.QUERY_UPDATE_LAST_AVAIL_PING_FORCE, query = "" //
+        + " UPDATE Agent a " //
+        + "    SET lastAvailabilityPing = :now " //
         + "  WHERE name = :agentName ") //
 
 })
@@ -191,6 +195,7 @@ public class Agent implements Serializable {
 
     public static final String QUERY_UPDATE_LAST_AVAIL_REPORT = "Agent.updateLastAvailReport";
     public static final String QUERY_UPDATE_LAST_AVAIL_PING = "Agent.updateLastAvailPing";
+    public static final String QUERY_UPDATE_LAST_AVAIL_PING_FORCE = "Agent.updateLastAvailPingForce";
 
     // this value is set, when authorized user wants to reset the token
     public static final String SECURITY_TOKEN_RESET = "@#$reset$#@";

modules/enterprise/agent/src/main/java/org/rhq/enterprise/agent/AgentMain.java

@@ -261,7 +261,7 @@ public class AgentMain {
     /**
      * Lock held when the m_clientSender needs to be created and destroyed.
      */
-     private final Object m_init = new Object();
+    private final Object m_init = new Object();
 
     /**
      * The time (as reported by <code>System.currentTimeMillis()</code>) when the agent was {@link #start() started}.
@@ -1884,7 +1884,8 @@ private void rebootPluginContainer() throws Throwable {
             LOG.debug(AgentI18NResourceKeys.CREATING_PLUGIN_CONTAINER_SERVER_SERVICES);
 
             //make the service container understand the requests for plugin container lifecycle events.
-            getServiceContainer().addRemotePojo(new PluginContainerLifecycleListener(this), PluginContainerLifecycle.class);
+            getServiceContainer().addRemotePojo(new PluginContainerLifecycleListener(this),
+                PluginContainerLifecycle.class);
 
             // Get remote pojo's for server access and make them accessible in the configuration object
             ClientRemotePojoFactory factory = m_clientSender.getClientRemotePojoFactory();
@@ -3787,6 +3788,18 @@ public void run() {
                 // take this opportunity to check the agent-server clock sync
                 serverClockNotification(request.getReplyServerTimestamp());
 
+                // If the server thinks we are down, we need to do some things to get this agent in sync
+                // with the server. Anything we do in here should be very fast.
+                boolean serverThinksWeAreDown = request.isReplyAgentIsBackfilled();
+                if (serverThinksWeAreDown) {
+                    LOG.warn(AgentI18NResourceKeys.SERVER_THINKS_AGENT_IS_DOWN);
+                    PluginContainer pluginContainer = PluginContainer.getInstance();
+                    if (pluginContainer.isStarted()) {
+                        // tell the plugin container to send a full avail report up so the server knows we are UP
+                        pluginContainer.getInventoryManager().requestFullAvailabilityReport();
+                    }
+                }
+
             } catch (Throwable t) {
                 // If the ping fails, typically do to a CannotConnectException, and we're not using autodiscovery,
                 // then start the poller to have sending mode re-established when the connection resumes.

modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerBean.java

@@ -798,7 +798,7 @@ public PingRequest handlePingRequest(PingRequest request) {
         long now = System.currentTimeMillis();
 
         if (request.isRequestUpdateAvailability()) {
-            updateLastAvailabilityPing(request.getAgentName(), now);
+            request.setReplyAgentIsBackfilled(updateLastAvailabilityPing(request.getAgentName(), now));
             request.setReplyUpdateAvailability(true);
         }
 
@@ -809,16 +809,31 @@ public PingRequest handlePingRequest(PingRequest request) {
         return request;
     }
 
-    private void updateLastAvailabilityPing(String agentName, long now) {
+    /**
+     * @return true if the agent is currently backfilled, false otherwise
+     */
+    private boolean updateLastAvailabilityPing(String agentName, long now) {
         /*
-         * since we already know we have to update the agent row with the last avail ping time, might as well
-         * set the backfilled to false here (as opposed to called agentManager.setBackfilled(agentId, false)
-         */
+          * There are two cases here: The agent is backfilled (rare) or not backfilled (typical).
+          * In both cases update the ping time.  If it is backfilled then reflect that fact
+          * in the response so the agent can correct the situation. Note that this takes care of
+          * the unusual case of BZ 1094540.
+          */
         Query query = entityManager.createNamedQuery(Agent.QUERY_UPDATE_LAST_AVAIL_PING);
         query.setParameter("now", now);
         query.setParameter("agentName", agentName);
 
-        query.executeUpdate();
+        int numUpdates = query.executeUpdate();
+        if (0 == numUpdates) {
+            query = entityManager.createNamedQuery(Agent.QUERY_UPDATE_LAST_AVAIL_PING_FORCE);
+            query.setParameter("now", now);
+            query.setParameter("agentName", agentName);
+            query.executeUpdate();
+
+            return true;
+        }
+
+        return false;
     }
 
     @ExcludeDefaultInterceptors