diff --git a/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala b/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala index 86aa103c209..02819ab6711 100644 --- a/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala +++ b/webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala @@ -117,18 +117,18 @@ class ReadOnlySoftwareDAOImpl( // fetch all softwares, for all nodes, in all 3 dits val acceptedDit = inventoryDitService.getDit(AcceptedInventory) - var mutSetSoftwares: Box[scala.collection.mutable.Set[SoftwareUuid]] = Full(scala.collection.mutable.Set[SoftwareUuid]()) + var mutSetSoftwares: scala.collection.mutable.Set[SoftwareUuid] = scala.collection.mutable.Set[SoftwareUuid]() val t1 = System.currentTimeMillis - for { + (for { con <- ldap // fetch all nodes - nodes = con.searchSub(acceptedDit.NODES.dn.getParent, IS(OC_NODE), A_NODE_UUID) + nodes = con.searchSub(acceptedDit.NODES.dn.getParent, IS(OC_NODE), A_NODE_UUID) - batchedNodes = nodes.grouped(50) + batchedNodes = nodes.grouped(50).toSeq - _ = batchedNodes.foreach { nodeEntries: Seq[LDAPEntry] => + _ <- sequence(batchedNodes) { nodeEntries: Seq[LDAPEntry] => val nodeIds = nodeEntries.flatMap(_(A_NODE_UUID)).map(NodeId(_)) val t2 = System.currentTimeMillis @@ -138,37 +138,17 @@ class ReadOnlySoftwareDAOImpl( val results = sequence(ids) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) } val t3 = System.currentTimeMillis() logger.debug(s"Software DNs from 50 nodes fetched in ${t3-t2}ms") - results match { + results match { // we don't want to return "results" because we need on-site dedup. case Full(softIds) => - mutSetSoftwares = mutSetSoftwares.map(t => t ++ softIds) + mutSetSoftwares = mutSetSoftwares ++ softIds Full(Unit) - case Failure(msg, exception, chain) => mutSetSoftwares = Failure(msg, exception, chain) // otherwise the time is wrong + case Failure(msg, exception, chain) => + Failure(msg, exception, chain) // otherwise the time is wrong } } - - - } yield { - mutSetSoftwares - } - mutSetSoftwares.map(x => x.toSet) - -/* - // TODO: This needs pagination, with 1000 nodes, it uses about 1,5 GB - softwareEntry = con.searchSub(acceptedDit.NODES.dn, IS(OC_NODE), A_SOFTWARE_DN) // it's really a dn that is stored in software attribute - t2 = System.currentTimeMillis() - _ = logger.debug(s"All Software DNs from all nodes ${softwareEntry.size} fetched in ${t2-t1}ms") - - // This could be more efficient, as it takes 3 secodes to process - ids = softwareEntry.flatMap( entry => entry.valuesFor(A_SOFTWARE_DN).toSet ) - t3 = System.currentTimeMillis() - _ = logger.debug(s"All software DNs deduplicated, resulting in ${ids.size} software entries, in ${t3-t2}ms") - - softIds <- sequence(ids.toSeq) { id => acceptedDit.SOFTWARE.SOFT.idFromDN(new DN(id)) } - t4 = System.currentTimeMillis() - _ = logger.trace(s"All software DNs to software ids in ${t4-t3}ms") - } yield { - softIds.toSet - }*/ + } yield { + () + }).map(_ => mutSetSoftwares.toSet) } } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala index bd4144c3e29..e5fb7698d9b 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/batch/PurgeUnreferencedSoftwares.scala @@ -61,7 +61,7 @@ class PurgeUnreferencedSoftwares( logger.info(s"Disable automatic purge of unreferenced softwares (update interval cannot be less than 1 hour)") } else { logger.debug(s"***** starting batch that purge unreferenced softwares, every ${updateInterval.toString()} *****") - scheduler.scheduleWithFixedDelay(12.second, updateInterval) { + scheduler.scheduleWithFixedDelay(1.hour, updateInterval) { softwareService.deleteUnreferencedSoftware() match { case Full(softwares) => logger.info(s"Purged ${softwares.length} unreferenced softwares") diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala index d43b59d15aa..ea3641f7fff 100644 --- a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala +++ b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala @@ -369,9 +369,7 @@ object RudderConfig extends Loggable { //val updateDynamicGroupsService : DynGroupUpdaterService = dynGroupUpdaterService val updateDynamicGroups: UpdateDynamicGroups = dyngroupUpdaterBatch val checkInventoryUpdate = new CheckInventoryUpdate(nodeInfoServiceImpl, asyncDeploymentAgent, stringUuidGenerator, 15.seconds) - val purgeUnreferencedSoftwares = new PurgeUnreferencedSoftwares(softwareService, RUDDER_BATCH_DELETE_SOFTWARE_INTERVAL.hours) - val databaseManager: DatabaseManager = databaseManagerImpl val automaticReportsCleaning: AutomaticReportsCleaning = dbCleaner val checkTechniqueLibrary: CheckTechniqueLibrary = techniqueLibraryUpdater @@ -1079,7 +1077,6 @@ object RudderConfig extends Loggable { private[this] lazy val databaseManagerImpl = new DatabaseManagerImpl(reportsRepositoryImpl, updateExpectedRepo) private[this] lazy val softwareInventoryDAO: ReadOnlySoftwareDAO = new ReadOnlySoftwareDAOImpl(inventoryDitService, roLdap, inventoryMapper) private[this] lazy val softwareInventoryRWDAO: WriteOnlySoftwareDAO = new WriteOnlySoftwareDAOImpl(acceptedNodesDitImpl, rwLdap) - private[this] lazy val softwareService: SoftwareService = new SoftwareServiceImpl(softwareInventoryDAO, softwareInventoryRWDAO) private[this] lazy val nodeSummaryServiceImpl = new NodeSummaryServiceImpl(inventoryDitService, inventoryMapper, roLdap)