From fe7bf28b22df50b1f8628e516eaeb3b0568ccee8 Mon Sep 17 00:00:00 2001 From: Nicolas Charles Date: Thu, 23 May 2019 13:49:38 +0200 Subject: [PATCH] Fixes #14924: Cleanup unreferenced software --- .../ReadOnlySoftwareInventoryDAOImpl.scala | 21 ++++++++------ .../inventory/ldap/core/SoftwareService.scala | 10 +++---- .../ldap-data/inventory-sample-data.ldif | 8 +++++ .../inventory/ldap/core/TestInventory.scala | 29 ++++++++++++++++++- .../batch/PurgeUnreferencedSoftwares.scala | 2 +- 5 files changed, 54 insertions(+), 16 deletions(-) diff --git a/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala b/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala index b4c6ee46543..86aa103c209 100644 --- a/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala +++ b/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala @@ -122,19 +122,22 @@ class ReadOnlySoftwareDAOImpl( val t1 = System.currentTimeMillis for { con <- ldap + // fetch all nodes nodes = con.searchSub(acceptedDit.NODES.dn.getParent, IS(OC_NODE), A_NODE_UUID) batchedNodes = nodes.grouped(50) _ = batchedNodes.foreach { nodeEntries: Seq[LDAPEntry] => - val nodeIds = nodeEntries.flatMap(_(A_NODE_UUID)).map(NodeId(_)) - - val orFilter: Filter = BuildFilter.OR(nodeIds.map(x => EQ(A_NODE_UUID, x.value)): _*) - val softwareEntry: Seq[LDAPEntry] = con.searchSub(acceptedDit.NODES.dn.getParent, orFilter, A_SOFTWARE_DN) - val ids: Seq[String] = softwareEntry.flatMap(entry => entry.valuesFor(A_SOFTWARE_DN).toSet ) - val results = sequence(ids) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) } - + val nodeIds = nodeEntries.flatMap(_(A_NODE_UUID)).map(NodeId(_)) + + val t2 = System.currentTimeMillis + val orFilter = BuildFilter.OR(nodeIds.map(x => EQ(A_NODE_UUID, x.value)): _*) + val softwareEntry= con.searchSub(acceptedDit.NODES.dn.getParent, orFilter, A_SOFTWARE_DN) + val ids = softwareEntry.flatMap(entry => entry.valuesFor(A_SOFTWARE_DN).toSet ) + val results = sequence(ids) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) } + val t3 = System.currentTimeMillis() + logger.debug(s"Software DNs from 50 nodes fetched in ${t3-t2}ms") results match { case Full(softIds) => mutSetSoftwares = mutSetSoftwares.map(t => t ++ softIds) @@ -149,9 +152,9 @@ class ReadOnlySoftwareDAOImpl( } mutSetSoftwares.map(x => x.toSet) - /* +/* // TODO: This needs pagination, with 1000 nodes, it uses about 1,5 GB - softwareEntry = dits.flatMap { dit => con.searchOne(dit.NODES.dn, IS(OC_NODE), A_SOFTWARE_DN) } // it's really a dn that is stored in software attribute + softwareEntry = con.searchSub(acceptedDit.NODES.dn, IS(OC_NODE), A_SOFTWARE_DN) // it's really a dn that is stored in software attribute t2 = System.currentTimeMillis() _ = logger.debug(s"All Software DNs from all nodes ${softwareEntry.size} fetched in ${t2-t1}ms") diff --git a/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/SoftwareService.scala b/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/SoftwareService.scala index f161be1998a..5f1527899c3 100644 --- a/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/SoftwareService.scala +++ b/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/SoftwareService.scala @@ -8,12 +8,12 @@ import net.liftweb.common._ trait SoftwareService { def deleteUnreferencedSoftware() : Box[Seq[String]] - } + class SoftwareServiceImpl( - readOnlySoftware: ReadOnlySoftwareDAO - , writeOnlySoftwareDAO: WriteOnlySoftwareDAO) - extends SoftwareService with Loggable { + readOnlySoftware : ReadOnlySoftwareDAO + , writeOnlySoftware : WriteOnlySoftwareDAO) +extends SoftwareService with Loggable { /** Delete all unreferenced softwares * First search in software, and then in nodes, so that if a node arrives in between (new inventory) @@ -34,7 +34,7 @@ class SoftwareServiceImpl( extraSoftware = allSoftwares -- allNodesSoftwares _ = logger.debug(s"Found ${extraSoftware.size} unreferenced software in ou=software, going to delete them") - deletedSoftware <- writeOnlySoftwareDAO.deleteSoftwares(extraSoftware.toSeq) + deletedSoftware <- writeOnlySoftware.deleteSoftwares(extraSoftware.toSeq) t4 = System.currentTimeMillis() _ = logger.debug(s"Deleted ${deletedSoftware.size} software in ${t4-t3}ms") diff --git a/webapp/sources/ldap-inventory/inventory-repository/src/test/resources/ldap-data/inventory-sample-data.ldif b/webapp/sources/ldap-inventory/inventory-repository/src/test/resources/ldap-data/inventory-sample-data.ldif index 92dbe618a6d..aa96e1d98f4 100644 --- a/webapp/sources/ldap-inventory/inventory-repository/src/test/resources/ldap-data/inventory-sample-data.ldif +++ b/webapp/sources/ldap-inventory/inventory-repository/src/test/resources/ldap-data/inventory-sample-data.ldif @@ -19,6 +19,14 @@ cn: Software 2 softwareVersion: 2.0-rc description: Second software +dn: softwareId=soft2,ou=Software,ou=Inventories,cn=rudder-configuration +softwareId: soft2 +objectClass: software +objectClass: top +cn: Software 3 +softwareVersion: 3.0-rc +description: Third software, unreferenced + ################################################################################################### # Nodes # ################################################################################################### diff --git a/webapp/sources/ldap-inventory/inventory-repository/src/test/scala/com/normation/inventory/ldap/core/TestInventory.scala b/webapp/sources/ldap-inventory/inventory-repository/src/test/scala/com/normation/inventory/ldap/core/TestInventory.scala index c4c83107afc..480099289d7 100644 --- a/webapp/sources/ldap-inventory/inventory-repository/src/test/scala/com/normation/inventory/ldap/core/TestInventory.scala +++ b/webapp/sources/ldap-inventory/inventory-repository/src/test/scala/com/normation/inventory/ldap/core/TestInventory.scala @@ -43,7 +43,7 @@ import org.specs2.runner._ import com.normation.ldap.listener.InMemoryDsConnectionProvider import com.unboundid.ldap.sdk.DN import com.normation.inventory.domain._ -import com.normation.ldap.sdk.RwLDAPConnection +import com.normation.ldap.sdk.{RoLDAPConnection, RwLDAPConnection} import net.liftweb.common.Empty import net.liftweb.common.Full import net.liftweb.common.EmptyBox @@ -93,6 +93,11 @@ class TestInventory extends Specification { , bootstrapLDIFPaths = bootstrapLDIFs ) + val roLdap = InMemoryDsConnectionProvider[RoLDAPConnection]( + baseDNs = baseDN :: Nil + , schemaLDIFPaths = schemaLDIFs + , bootstrapLDIFPaths = bootstrapLDIFs + ) val softwareDN = new DN("ou=Inventories, cn=rudder-configuration") @@ -117,6 +122,12 @@ class TestInventory extends Specification { val repo = new FullInventoryRepositoryImpl(inventoryDitService, inventoryMapper, ldap) + val readOnlySoftware = new ReadOnlySoftwareDAOImpl(inventoryDitService, roLdap, inventoryMapper) + + val writeOnlySoftware = new WriteOnlySoftwareDAOImpl(acceptedNodesDitImpl, ldap) + + + val softwareService = new SoftwareServiceImpl(readOnlySoftware, writeOnlySoftware) val allStatus = Seq(RemovedInventory, PendingInventory, AcceptedInventory) @@ -400,6 +411,22 @@ class TestInventory extends Specification { } + "Softwares" should { + "Find 2 software referenced by nodes with the repository" in { + val softwares = readOnlySoftware.getSoftwaresForAllNodes() + softwares.isOK and softwares.map(_.size) == Full(2) + } + + "Find 3 software in ou=software with the repository" in { + val softwares = readOnlySoftware.getAllSoftwareIds() + softwares.isOK and softwares.map(_.size) == Full(3) + } + + "Purge one unreferenced software with the SoftwareService" in { + val purgedSoftwares = softwareService.deleteUnreferencedSoftware() + purgedSoftwares.isOK and purgedSoftwares.map(_.size) == Full(1) + } + } step { ldap.close diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala index 9a1154d9b6a..bd4144c3e29 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala @@ -61,7 +61,7 @@ class PurgeUnreferencedSoftwares( logger.info(s"Disable automatic purge of unreferenced softwares (update interval cannot be less than 1 hour)") } else { logger.debug(s"***** starting batch that purge unreferenced softwares, every ${updateInterval.toString()} *****") - scheduler.scheduleWithFixedDelay(updateInterval, updateInterval) { + scheduler.scheduleWithFixedDelay(12.second, updateInterval) { softwareService.deleteUnreferencedSoftware() match { case Full(softwares) => logger.info(s"Purged ${softwares.length} unreferenced softwares")