Fixes #15675: Leak in Cache of Node Compliance and NodeInfo and perfs improvement

webapp/sources/rudder/pom.xml

@@ -59,7 +59,14 @@ along with Rudder.  If not, see <http://www.gnu.org/licenses/>.
     <!-- fill-in rule templates -->
   </modules>
 
-  <dependencies/>
+  <dependencies>
+	  <dependency>
+	  <groupId>com.github.jbellis</groupId>
+	  <artifactId>jamm</artifactId>
+	  <version>0.3.3</version>
+	  <scope>provided</scope>
+  </dependency>
+  </dependencies>
 
   <repositories>
     <repository>

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/reports/execution/ReportsExecutionRepository.scala

@@ -117,7 +117,7 @@ class CachedReportsExecutionRepository(
     cache = Map()
   }
 
-  override def getNodesLastRun(nodeIds: Set[NodeId]): Box[Map[NodeId, Option[AgentRunWithNodeConfig]]] = this.synchronized {
+  override def getNodesLastRun(nodeIds: Set[NodeId]): Box[Map[NodeId, Option[AgentRunWithNodeConfig]]] = scala.concurrent.blocking { this.synchronized {
     val n1 = System.currentTimeMillis
     (for {
       runs <- readBackend.getNodesLastRun(nodeIds.diff(cache.keySet))
@@ -127,7 +127,7 @@ class CachedReportsExecutionRepository(
       cache = cache ++ runs
       cache.filterKeys { x => nodeIds.contains(x) }
     }) ?~! s"Error when trying to update the cache of Agent Runs informations"
-  }
+  } }
 
   override def updateExecutions(executions : Seq[AgentRun]) : Seq[Box[AgentRun]] = this.synchronized {
     logger.trace(s"Update runs for nodes [${executions.map( _.agentRunId.nodeId.value ).mkString(", ")}]")

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/DataStructures.scala

@@ -357,7 +357,7 @@ final case class Policy(
   // == .toList (keep order)     ==> List[List[Variable]]
   // == flatten (keep order)     ==> List[Variable]
   val expandedVars    = Policy.mergeVars(policyVars.map( _.expandedVars.values).toList.flatten)
-  val originalVars    = Policy.mergeVars(policyVars.map( _.originalVars.values).toList.flatten)
+  //val originalVars    = Policy.mergeVars(policyVars.map( _.originalVars.values).toList.flatten)
   val trackerVariable = policyVars.head.trackerVariable.spec.cloneSetMultivalued.toVariable(policyVars.map(_.trackerVariable.values).toList.flatten)
 }
 

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationLogger.scala

@@ -55,6 +55,8 @@ trait NodeConfigurationLogger {
 
   def log(nodeConfiguration: Seq[NodeConfiguration]): Box[Set[NodeId]]
 
+  def isDebugEnabled: Boolean
+
 }
 
 /**
@@ -70,6 +72,8 @@ class NodeConfigurationLoggerImpl(
 
   val logger = LoggerFactory.getLogger("rudder.debug.nodeconfiguration")
 
+  def isDebugEnabled: Boolean = logger.isDebugEnabled
+
   {
     val p = new File(path)
     p.mkdirs()

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala

@@ -47,7 +47,7 @@ import com.normation.inventory.domain.AgentType.CfeEnterprise
 import com.normation.inventory.domain.NodeId
 import com.normation.rudder.domain.Constants
 import com.normation.rudder.domain.licenses.CfeEnterpriseLicense
-import com.normation.rudder.domain.nodes.NodeProperty
+import com.normation.rudder.domain.nodes.{NodeInfo, NodeProperty}
 import com.normation.rudder.domain.policies.GlobalPolicyMode
 import com.normation.rudder.domain.reports.NodeConfigId
 import com.normation.rudder.hooks.HookEnvPairs
@@ -97,15 +97,16 @@ trait PolicyWriterService {
    *
    */
   def writeTemplate(
-      rootNodeId    : NodeId
-    , nodesToWrite  : Set[NodeId]
-    , allNodeConfigs: Map[NodeId, NodeConfiguration]
-    , versions      : Map[NodeId, NodeConfigId]
-    , allLicenses   : Map[NodeId, CfeEnterpriseLicense]
-    , globalPolicyMode: GlobalPolicyMode
-    , generationTime  : DateTime
-    , parallelism     : Parallelism
-  ) : Box[Seq[NodeConfiguration]]
+      rootNodeId        : NodeId
+    , nodesToWrite      : Set[NodeId]
+    , updatedNodeConfigs: Map[NodeId, NodeConfiguration]
+    , allNodeInfos      : Map[NodeId, NodeInfo]
+    , versions          : Map[NodeId, NodeConfigId]
+    , allLicenses       : Map[NodeId, CfeEnterpriseLicense]
+    , globalPolicyMode  : GlobalPolicyMode
+    , generationTime    : DateTime
+    , parallelism       : Parallelism
+  ) : Box[Seq[NodeId]]
 }
 
 class PolicyWriterServiceImpl(
@@ -147,7 +148,7 @@ class PolicyWriterServiceImpl(
         val message = s"could not write ${fileName} file, cause is: ${e.getMessage}"
         Failure(message)
       case Success(_) =>
-        Full(agentNodeConfig)
+        Full(agentNodeConfig.config.nodeInfo.id)
     }
   }
 
@@ -170,7 +171,7 @@ class PolicyWriterServiceImpl(
         val message = s"could not write ${fileName} file, cause is: ${e.getMessage}"
         Failure(message)
       case Success(_) =>
-        Full(agentNodeConfig)
+        Full(agentNodeConfig.config.nodeInfo.id)
     }
   }
 
@@ -212,29 +213,31 @@ class PolicyWriterServiceImpl(
    *
    */
   override def writeTemplate(
-      rootNodeId      : NodeId
-    , nodesToWrite    : Set[NodeId]
-    , allNodeConfigs  : Map[NodeId, NodeConfiguration]
-    , versions        : Map[NodeId, NodeConfigId]
-    , allLicenses     : Map[NodeId, CfeEnterpriseLicense]
-    , globalPolicyMode: GlobalPolicyMode
-    , generationTime  : DateTime
-    , parallelism     : Parallelism
-  ) : Box[Seq[NodeConfiguration]] = {
-
-    val nodeConfigsToWrite     = allNodeConfigs.filterKeys(nodesToWrite.contains(_))
-    val interestingNodeConfigs = allNodeConfigs.filterKeys(k => nodeConfigsToWrite.exists{ case(x, _) => x == k }).values.toSeq
-    val techniqueIds           = interestingNodeConfigs.flatMap( _.getTechniqueIds ).toSet
-
-    //debug - but don't fails for debugging !
-    logNodeConfig.log(nodeConfigsToWrite.values.toSeq) match {
-      case eb:EmptyBox =>
-        val e = eb ?~! "Error when trying to write node configurations for debugging"
-        logger.error(e)
-        e.rootExceptionCause.foreach { ex =>
-          logger.error("Root exception cause was:", ex)
+      rootNodeId        : NodeId
+    , nodesToWrite      : Set[NodeId]
+    , updatedNodeConfigs: Map[NodeId, NodeConfiguration]
+    , allNodeInfos      : Map[NodeId, NodeInfo]
+    , versions          : Map[NodeId, NodeConfigId]
+    , allLicenses       : Map[NodeId, CfeEnterpriseLicense]
+    , globalPolicyMode  : GlobalPolicyMode
+    , generationTime    : DateTime
+    , parallelism       : Parallelism
+  ) : Box[Seq[NodeId]] = {
+
+    val interestingNodeConfigs = updatedNodeConfigs.values.toSeq
+    val techniqueIds = interestingNodeConfigs.flatMap(_.getTechniqueIds).toSet
+
+    if (logNodeConfig.isDebugEnabled) {
+      //debug - but don't fails for debugging !
+      logNodeConfig.log(interestingNodeConfigs) match {
+        case eb: EmptyBox =>
+          val e = eb ?~! "Error when trying to write node configurations for debugging"
+          logger.error(e)
+          e.rootExceptionCause.foreach { ex =>
+            logger.error("Root exception cause was:", ex)
+          }
+          case _ => //nothing to do
         }
-      case _ => //nothing to do
     }
 
     /*
@@ -265,7 +268,7 @@ class PolicyWriterServiceImpl(
 
     val readTemplateTime1 = System.currentTimeMillis
     for {
-      configAndPaths   <- calculatePathsForNodeConfigurations(interestingNodeConfigs, rootNodeId, allNodeConfigs, newPostfix, backupPostfix)
+      configAndPaths   <- calculatePathsForNodeConfigurations(interestingNodeConfigs, rootNodeId, allNodeInfos, newPostfix, backupPostfix)
       pathsInfo        =  configAndPaths.map { _.paths }
       templates        <- readTemplateFromFileSystem(techniqueIds)
       resources        <- readResourcesFromFileSystem(techniqueIds)
@@ -278,10 +281,9 @@ class PolicyWriterServiceImpl(
       //////////
       // nothing agent specific before that
       //////////
-
       preparedPromises <- ParallelSequence.traverse(configAndPaths) { case agentNodeConfig =>
                             val nodeConfigId = versions(agentNodeConfig.config.nodeInfo.id)
-                            prepareTemplate.prepareTemplateForAgentNodeConfiguration(agentNodeConfig, nodeConfigId, rootNodeId, templates, allNodeConfigs, Policy.TAG_OF_RUDDER_ID, globalPolicyMode, generationTime) ?~!
+                            prepareTemplate.prepareTemplateForAgentNodeConfiguration(agentNodeConfig, nodeConfigId, rootNodeId, templates, Policy.TAG_OF_RUDDER_ID, globalPolicyMode, generationTime) ?~!
                             s"Error when calculating configuration for node '${agentNodeConfig.config.nodeInfo.hostname}' (${agentNodeConfig.config.nodeInfo.id.value})"
                          }
       preparedPromisesTime = System.currentTimeMillis
@@ -306,7 +308,7 @@ class PolicyWriterServiceImpl(
       // nothing agent specific after that
       //////////
 
-      propertiesWritten <- ParallelSequence.traverse(configAndPaths) { case agentNodeConfig =>
+      _                   <- ParallelSequence.traverse(configAndPaths) { case agentNodeConfig =>
                              writeNodePropertiesFile(agentNodeConfig) ?~!
                                s"An error occured while writing property file for Node ${agentNodeConfig.config.nodeInfo.hostname} (id: ${agentNodeConfig.config.nodeInfo.id.value}"
                            }
@@ -315,7 +317,7 @@ class PolicyWriterServiceImpl(
       propertiesWrittenDur  = propertiesWrittenTime - promiseWrittenTime
       _                     = policyLogger.debug(s"Properties written in ${propertiesWrittenDur} ms")
 
-      parametersWritten <- ParallelSequence.traverse(configAndPaths) { case agentNodeConfig =>
+      _                   <- ParallelSequence.traverse(configAndPaths) { case agentNodeConfig =>
                              writeRudderParameterFile(agentNodeConfig) ?~!
                                s"An error occured while writing parameter file for Node ${agentNodeConfig.config.nodeInfo.hostname} (id: ${agentNodeConfig.config.nodeInfo.id.value}"
                           }
@@ -324,12 +326,14 @@ class PolicyWriterServiceImpl(
       parametersWrittenDur  = parametersWrittenTime - propertiesWrittenTime
       _                     = policyLogger.debug(s"Parameters written in ${parametersWrittenDur} ms")
 
-      licensesCopied   <- copyLicenses(configAndPaths, allLicenses)
+      _                  <- copyLicenses(configAndPaths, allLicenses)
 
       licensesCopiedTime = System.currentTimeMillis
       licensesCopiedDur  = licensesCopiedTime - parametersWrittenTime
       _                  = policyLogger.debug(s"Licenses copied in ${licensesCopiedDur} ms")
 
+
+
       _                 = fillTemplates.clearCache
       /// perhaps that should be a post-hook somehow ?
       // and perhaps we should have an AgentSpecific global pre/post write
@@ -397,8 +401,7 @@ class PolicyWriterServiceImpl(
       postMvHooksTime2   = System.currentTimeMillis
       _                  = policyLogger.debug(s"Hooks for policy-generation-node-finished executed in ${postMvHooksTime2 - movedPromisesTime2} ms")
     } yield {
-      val ids = movedPromises.map { _.nodeId }.toSet
-      allNodeConfigs.filterKeys { id => ids.contains(id) }.values.toSeq
+      movedPromises.map { _.nodeId }
     }
 
   }
@@ -415,7 +418,7 @@ class PolicyWriterServiceImpl(
   private[this] def calculatePathsForNodeConfigurations(
       configs            : Seq[NodeConfiguration]
     , rootNodeConfigId   : NodeId
-    , allNodeConfigs     : Map[NodeId, NodeConfiguration]
+    , allNodeInfos       : Map[NodeId, NodeInfo]
     , newsFileExtension  : String
     , backupFileExtension: String
   ): Box[Seq[AgentNodeConfiguration]] = {
@@ -438,7 +441,7 @@ class PolicyWriterServiceImpl(
                       , pathComputer.getRootPath(agentType) + backupFileExtension
                     ))
                   } else {
-                    pathComputer.computeBaseNodePath(config.nodeInfo.id, rootNodeConfigId, allNodeConfigs.mapValues(_.nodeInfo)).map { case NodePromisesPaths(id, base, news, backup) =>
+                    pathComputer.computeBaseNodePath(config.nodeInfo.id, rootNodeConfigId, allNodeInfos).map { case NodePromisesPaths(id, base, news, backup) =>
                         val postfix = agentType.toRulesPath
                         NodePromisesPaths(id, base + postfix, news + postfix, backup + postfix)
                     }
@@ -760,8 +763,6 @@ class PolicyWriterServiceImpl(
 
   /**
    * Move the machine promises folder  to the backup folder
-   * @param machineFolder
-   * @param backupFolder
    */
   private[this] def backupNodeFolder(nodeFolder: String, backupFolder: String): Unit = {
     val src = new File(nodeFolder)
@@ -777,8 +778,6 @@ class PolicyWriterServiceImpl(
 
   /**
    * Move the newly created folder to the final location
-   * @param newFolder : where the promises have been written
-   * @param nodeFolder : where the promises will be
    */
   private[this] def moveNewNodeFolder(sourceFolder: String, destinationFolder: String): Unit = {
     val src = new File(sourceFolder)

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariables.scala

@@ -72,7 +72,6 @@ trait PrepareTemplateVariables {
     , nodeConfigVersion: NodeConfigId
     , rootNodeConfigId : NodeId
     , templates        : Map[(TechniqueResourceId, AgentType), TechniqueTemplateCopyInfo]
-    , allNodeConfigs   : Map[NodeId, NodeConfiguration]
     , rudderIdCsvTag   : String
     , globalPolicyMode : GlobalPolicyMode
     , generationTime   : DateTime
@@ -97,7 +96,6 @@ class PrepareTemplateVariablesImpl(
     , nodeConfigVersion: NodeConfigId
     , rootNodeConfigId : NodeId
     , templates        : Map[(TechniqueResourceId, AgentType), TechniqueTemplateCopyInfo]
-    , allNodeConfigs   : Map[NodeId, NodeConfiguration]
     , rudderIdCsvTag   : String
     , globalPolicyMode : GlobalPolicyMode
     , generationTime   : DateTime

webapp/sources/rudder/rudder-web/pom.xml

@@ -88,6 +88,12 @@ along with Rudder.  If not, see <http://www.gnu.org/licenses/>.
               </includes>
             </resource>
           </webResources>
+
+	  <!-- this is needed to get instrumentation working, jar need to be out of web-inf -->
+          <packagingExcludes>
+            WEB-INF/lib/jamm-*.jar
+          </packagingExcludes>
+
         </configuration>
       </plugin>
       <plugin>