Fixes #14924: Cleanup unreferenced software

Normation · May 23, 2019 · fe7bf28 · fe7bf28
1 parent c5a2a83
commit fe7bf28
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 16 deletions.
diff --git a/...y/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala b/...y/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala
@@ -122,19 +122,22 @@ class ReadOnlySoftwareDAOImpl(
     val t1 = System.currentTimeMillis
     for {
       con           <- ldap
+
       // fetch all nodes
       nodes         = con.searchSub(acceptedDit.NODES.dn.getParent, IS(OC_NODE), A_NODE_UUID)
 
       batchedNodes = nodes.grouped(50)
 
       _ = batchedNodes.foreach { nodeEntries: Seq[LDAPEntry] =>
-                             val nodeIds = nodeEntries.flatMap(_(A_NODE_UUID)).map(NodeId(_))
-
-                             val orFilter: Filter = BuildFilter.OR(nodeIds.map(x => EQ(A_NODE_UUID, x.value)): _*)
-                             val softwareEntry: Seq[LDAPEntry] =  con.searchSub(acceptedDit.NODES.dn.getParent, orFilter, A_SOFTWARE_DN)
-                             val ids: Seq[String] = softwareEntry.flatMap(entry => entry.valuesFor(A_SOFTWARE_DN).toSet )
-                             val results = sequence(ids) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) }
-
+                             val nodeIds      = nodeEntries.flatMap(_(A_NODE_UUID)).map(NodeId(_))
+
+                             val t2           = System.currentTimeMillis
+                             val orFilter     = BuildFilter.OR(nodeIds.map(x => EQ(A_NODE_UUID, x.value)): _*)
+                             val softwareEntry=  con.searchSub(acceptedDit.NODES.dn.getParent, orFilter, A_SOFTWARE_DN)
+                             val ids          = softwareEntry.flatMap(entry => entry.valuesFor(A_SOFTWARE_DN).toSet )
+                             val results      = sequence(ids) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) }
+                             val t3           = System.currentTimeMillis()
+                             logger.debug(s"Software DNs from 50 nodes fetched in ${t3-t2}ms")
                              results match {
                                case Full(softIds) =>
                                  mutSetSoftwares = mutSetSoftwares.map(t => t ++ softIds)
@@ -149,9 +152,9 @@ class ReadOnlySoftwareDAOImpl(
       }
     mutSetSoftwares.map(x => x.toSet)
 
-    /*
+/*
     // TODO: This needs pagination, with 1000 nodes, it uses about 1,5 GB
-    softwareEntry =  dits.flatMap { dit => con.searchOne(dit.NODES.dn, IS(OC_NODE), A_SOFTWARE_DN) } // it's really a dn that is stored in software attribute
+    softwareEntry =  con.searchSub(acceptedDit.NODES.dn, IS(OC_NODE), A_SOFTWARE_DN) // it's really a dn that is stored in software attribute
     t2            = System.currentTimeMillis()
     _             = logger.debug(s"All Software DNs from all nodes ${softwareEntry.size} fetched in ${t2-t1}ms")
 

diff --git a/...ventory-repository/src/main/scala/com/normation/inventory/ldap/core/SoftwareService.scala b/...ventory-repository/src/main/scala/com/normation/inventory/ldap/core/SoftwareService.scala
@@ -8,12 +8,12 @@ import net.liftweb.common._
 
 trait SoftwareService {
   def deleteUnreferencedSoftware() : Box[Seq[String]]
-
 }
+
 class SoftwareServiceImpl(
-                           readOnlySoftware: ReadOnlySoftwareDAO
-                           , writeOnlySoftwareDAO: WriteOnlySoftwareDAO)
-  extends SoftwareService with Loggable {
+    readOnlySoftware    : ReadOnlySoftwareDAO
+  , writeOnlySoftware   : WriteOnlySoftwareDAO)
+extends SoftwareService with Loggable {
 
   /** Delete all unreferenced softwares
     * First search in software, and then in nodes, so that if a node arrives in between (new inventory)
@@ -34,7 +34,7 @@ class SoftwareServiceImpl(
       extraSoftware     = allSoftwares -- allNodesSoftwares
       _                 = logger.debug(s"Found ${extraSoftware.size} unreferenced software in ou=software, going to delete them")
 
-      deletedSoftware   <- writeOnlySoftwareDAO.deleteSoftwares(extraSoftware.toSeq)
+      deletedSoftware   <- writeOnlySoftware.deleteSoftwares(extraSoftware.toSeq)
       t4                = System.currentTimeMillis()
       _                 = logger.debug(s"Deleted ${deletedSoftware.size} software in ${t4-t3}ms")
 

diff --git a/...ap-inventory/inventory-repository/src/test/resources/ldap-data/inventory-sample-data.ldif b/...ap-inventory/inventory-repository/src/test/resources/ldap-data/inventory-sample-data.ldif
@@ -19,6 +19,14 @@ cn: Software 2
 softwareVersion: 2.0-rc
 description: Second software
 
+dn: softwareId=soft2,ou=Software,ou=Inventories,cn=rudder-configuration
+softwareId: soft2
+objectClass: software
+objectClass: top
+cn: Software 3
+softwareVersion: 3.0-rc
+description: Third software, unreferenced
+
 ###################################################################################################
 # Nodes #
 ###################################################################################################

diff --git a/...inventory-repository/src/test/scala/com/normation/inventory/ldap/core/TestInventory.scala b/...inventory-repository/src/test/scala/com/normation/inventory/ldap/core/TestInventory.scala
@@ -43,7 +43,7 @@ import org.specs2.runner._
 import com.normation.ldap.listener.InMemoryDsConnectionProvider
 import com.unboundid.ldap.sdk.DN
 import com.normation.inventory.domain._
-import com.normation.ldap.sdk.RwLDAPConnection
+import com.normation.ldap.sdk.{RoLDAPConnection, RwLDAPConnection}
 import net.liftweb.common.Empty
 import net.liftweb.common.Full
 import net.liftweb.common.EmptyBox
@@ -93,6 +93,11 @@ class TestInventory extends Specification {
     , bootstrapLDIFPaths = bootstrapLDIFs
   )
 
+  val roLdap = InMemoryDsConnectionProvider[RoLDAPConnection](
+    baseDNs = baseDN :: Nil
+    , schemaLDIFPaths = schemaLDIFs
+    , bootstrapLDIFPaths = bootstrapLDIFs
+  )
 
   val softwareDN = new DN("ou=Inventories, cn=rudder-configuration")
 
@@ -117,6 +122,12 @@ class TestInventory extends Specification {
 
   val repo = new FullInventoryRepositoryImpl(inventoryDitService, inventoryMapper, ldap)
 
+  val readOnlySoftware = new ReadOnlySoftwareDAOImpl(inventoryDitService, roLdap, inventoryMapper)
+
+  val writeOnlySoftware = new WriteOnlySoftwareDAOImpl(acceptedNodesDitImpl, ldap)
+
+
+  val softwareService = new SoftwareServiceImpl(readOnlySoftware, writeOnlySoftware)
 
   val allStatus = Seq(RemovedInventory, PendingInventory, AcceptedInventory)
 
@@ -400,6 +411,22 @@ class TestInventory extends Specification {
 
   }
 
+  "Softwares" should {
+    "Find 2 software referenced by nodes with the repository" in {
+      val softwares = readOnlySoftware.getSoftwaresForAllNodes()
+      softwares.isOK and softwares.map(_.size) == Full(2)
+    }
+
+    "Find 3 software in ou=software with the repository" in {
+      val softwares = readOnlySoftware.getAllSoftwareIds()
+      softwares.isOK and softwares.map(_.size) == Full(3)
+    }
+
+    "Purge one unreferenced software with the SoftwareService" in {
+      val purgedSoftwares = softwareService.deleteUnreferencedSoftware()
+      purgedSoftwares.isOK and purgedSoftwares.map(_.size) == Full(1)
+    }
+  }
 
   step {
     ldap.close

diff --git a/...er/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala b/...er/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala
@@ -61,7 +61,7 @@ class PurgeUnreferencedSoftwares(
     logger.info(s"Disable automatic purge of unreferenced softwares (update interval cannot be less than 1 hour)")
   } else {
     logger.debug(s"***** starting batch that purge unreferenced softwares, every ${updateInterval.toString()} *****")
-    scheduler.scheduleWithFixedDelay(updateInterval, updateInterval) {
+    scheduler.scheduleWithFixedDelay(12.second, updateInterval) {
       softwareService.deleteUnreferencedSoftware() match {
         case Full(softwares) =>
           logger.info(s"Purged ${softwares.length} unreferenced softwares")