Fixes #14924: Cleanup unreferenced software

Normation · Jun 11, 2019 · fd62862 · fd62862
2 parents 4517d59 + bd25a22
commit fd62862
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 36 deletions.
diff --git a/...y/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala b/...y/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala
@@ -117,18 +117,18 @@ class ReadOnlySoftwareDAOImpl(
     // fetch all softwares, for all nodes, in all 3 dits
     val acceptedDit = inventoryDitService.getDit(AcceptedInventory)
 
-    var mutSetSoftwares: Box[scala.collection.mutable.Set[SoftwareUuid]] = Full(scala.collection.mutable.Set[SoftwareUuid]())
+    var mutSetSoftwares: scala.collection.mutable.Set[SoftwareUuid] = scala.collection.mutable.Set[SoftwareUuid]()
 
     val t1 = System.currentTimeMillis
-    for {
+    (for {
       con           <- ldap
 
       // fetch all nodes
-      nodes         = con.searchSub(acceptedDit.NODES.dn.getParent, IS(OC_NODE), A_NODE_UUID)
+      nodes        = con.searchSub(acceptedDit.NODES.dn.getParent, IS(OC_NODE), A_NODE_UUID)
 
-      batchedNodes = nodes.grouped(50)
+      batchedNodes = nodes.grouped(50).toSeq
 
-      _ = batchedNodes.foreach { nodeEntries: Seq[LDAPEntry] =>
+      _            <- sequence(batchedNodes) { nodeEntries: Seq[LDAPEntry] =>
                              val nodeIds      = nodeEntries.flatMap(_(A_NODE_UUID)).map(NodeId(_))
 
                              val t2           = System.currentTimeMillis
@@ -138,37 +138,17 @@ class ReadOnlySoftwareDAOImpl(
                              val results      = sequence(ids) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) }
                              val t3           = System.currentTimeMillis()
                              logger.debug(s"Software DNs from 50 nodes fetched in ${t3-t2}ms")
-                             results match {
+                             results match { // we don't want to return "results" because we need on-site dedup.
                                case Full(softIds) =>
-                                 mutSetSoftwares = mutSetSoftwares.map(t => t ++ softIds)
+                                 mutSetSoftwares = mutSetSoftwares ++ softIds
                                  Full(Unit)
-                               case Failure(msg, exception, chain) => mutSetSoftwares = Failure(msg, exception, chain) // otherwise the time is wrong
+                               case Failure(msg, exception, chain) =>
+                                 Failure(msg, exception, chain) // otherwise the time is wrong
                              }
                            }
-
-
-      } yield {
-        mutSetSoftwares
-      }
-    mutSetSoftwares.map(x => x.toSet)
-
-/*
-    // TODO: This needs pagination, with 1000 nodes, it uses about 1,5 GB
-    softwareEntry =  con.searchSub(acceptedDit.NODES.dn, IS(OC_NODE), A_SOFTWARE_DN) // it's really a dn that is stored in software attribute
-    t2            = System.currentTimeMillis()
-    _             = logger.debug(s"All Software DNs from all nodes ${softwareEntry.size} fetched in ${t2-t1}ms")
-
-    // This could be more efficient, as it takes 3 secodes to process
-    ids           = softwareEntry.flatMap( entry => entry.valuesFor(A_SOFTWARE_DN).toSet )
-    t3            = System.currentTimeMillis()
-     _            = logger.debug(s"All software DNs deduplicated, resulting in ${ids.size} software entries, in ${t3-t2}ms")
-
-    softIds       <- sequence(ids.toSeq) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) }
-    t4            = System.currentTimeMillis()
-    _             = logger.trace(s"All software DNs to software ids in ${t4-t3}ms")
-  } yield {
-    softIds.toSet
-  }*/
+    } yield {
+      ()
+    }).map(_ => mutSetSoftwares.toSet)
   }
 
 }

diff --git a/...er/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala b/...er/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala
@@ -61,7 +61,7 @@ class PurgeUnreferencedSoftwares(
     logger.info(s"Disable automatic purge of unreferenced softwares (update interval cannot be less than 1 hour)")
   } else {
     logger.debug(s"***** starting batch that purge unreferenced softwares, every ${updateInterval.toString()} *****")
-    scheduler.scheduleWithFixedDelay(12.second, updateInterval) {
+    scheduler.scheduleWithFixedDelay(1.hour, updateInterval) {
       softwareService.deleteUnreferencedSoftware() match {
         case Full(softwares) =>
           logger.info(s"Purged ${softwares.length} unreferenced softwares")

diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala
@@ -369,9 +369,7 @@ object RudderConfig extends Loggable {
   //val updateDynamicGroupsService : DynGroupUpdaterService = dynGroupUpdaterService
   val updateDynamicGroups: UpdateDynamicGroups = dyngroupUpdaterBatch
   val checkInventoryUpdate = new CheckInventoryUpdate(nodeInfoServiceImpl, asyncDeploymentAgent, stringUuidGenerator, 15.seconds)
-
   val purgeUnreferencedSoftwares = new PurgeUnreferencedSoftwares(softwareService, RUDDER_BATCH_DELETE_SOFTWARE_INTERVAL.hours)
-
   val databaseManager: DatabaseManager = databaseManagerImpl
   val automaticReportsCleaning: AutomaticReportsCleaning = dbCleaner
   val checkTechniqueLibrary: CheckTechniqueLibrary = techniqueLibraryUpdater
@@ -1079,7 +1077,6 @@ object RudderConfig extends Loggable {
   private[this] lazy val databaseManagerImpl = new DatabaseManagerImpl(reportsRepositoryImpl, updateExpectedRepo)
   private[this] lazy val softwareInventoryDAO: ReadOnlySoftwareDAO = new ReadOnlySoftwareDAOImpl(inventoryDitService, roLdap, inventoryMapper)
   private[this] lazy val softwareInventoryRWDAO: WriteOnlySoftwareDAO = new WriteOnlySoftwareDAOImpl(acceptedNodesDitImpl, rwLdap)
-
   private[this] lazy val softwareService: SoftwareService = new SoftwareServiceImpl(softwareInventoryDAO, softwareInventoryRWDAO)
 
   private[this] lazy val nodeSummaryServiceImpl = new NodeSummaryServiceImpl(inventoryDitService, inventoryMapper, roLdap)