diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala index 7695e7e5bc8..5d5044e9e6e 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala @@ -170,8 +170,8 @@ object Doobie { implicit val DateTimeMeta: Meta[DateTime] = Meta[java.sql.Timestamp].imap(ts => new DateTime(ts.getTime()))(dt => new java.sql.Timestamp(dt.getMillis)) - implicit val ReadRuleId: Read[RuleId] = { - Read[String].map(r => { + implicit val ReadRuleId: Get[RuleId] = { + Get[String].map(r => { RuleId.parse(r) match { case Right(rid) => rid case Left(err) => @@ -179,11 +179,11 @@ object Doobie { } }) } - implicit val WriteRuleId: Write[RuleId] = { - Write[String].contramap(_.serialize) + implicit val PutRuleId: Put[RuleId] = { + Put[String].contramap(_.serialize) } - implicit val ReadDirectiveId: Read[DirectiveId] = { - Read[String].map(r => { + implicit val ReadDirectiveId: Get[DirectiveId] = { + Get[String].map(r => { DirectiveId.parse(r) match { case Right(rid) => rid case Left(err) => @@ -193,8 +193,8 @@ object Doobie { } }) } - implicit val WriteDirectiveId: Write[DirectiveId] = { - Write[String].contramap(_.serialize) + implicit val WriteDirectiveId: Put[DirectiveId] = { + Put[String].contramap(_.serialize) } implicit val ReportRead: Read[Reports] = { diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala index 993e14f9bdd..c56e009ef59 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala @@ -153,6 +153,20 @@ object NodeStatusReport { nodeStatusReport.reports.filter(r => ruleIds.contains(r.ruleId)) ) } + + def filterByDirectives(nodeStatusReport: NodeStatusReport, directiveIds: Set[DirectiveId]): NodeStatusReport = { + + new NodeStatusReport( + nodeStatusReport.nodeId, + nodeStatusReport.runInfo, + nodeStatusReport.statusInfo, + nodeStatusReport.overrides, + nodeStatusReport.reports.flatMap{r => + val filterRule = r.copy(directives = r.directives.filter(d => directiveIds.contains(d._1))) + if (filterRule.directives.isEmpty) None else Some (filterRule) + } + ) + } } /** diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala index 865df611234..0321033f009 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala @@ -39,6 +39,7 @@ package com.normation.rudder.repository import com.normation.box._ import com.normation.errors._ import com.normation.inventory.domain.NodeId +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ import net.liftweb.common.Box @@ -109,6 +110,14 @@ trait FindExpectedReportRepository { */ def findCurrentNodeIdsForRule(ruleId: RuleId, nodeIds: Set[NodeId]): IOResult[Set[NodeId]] + + /** + * Return node ids associated to the rule (based on expectedreports (the one still pending)) for this Rule, + * only limited on the nodeIds in parameter (used when cache is incomplete) + */ + def findCurrentNodeIdsForDirective(ruleId: DirectiveId, nodeIds: Set[NodeId]): IOResult[Set[NodeId]] + def findCurrentNodeIdsForDirective(ruleId: DirectiveId): IOResult[Set[NodeId]] + /* * Retrieve the expected reports by config version of the nodes. * diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala index 92f590c535c..1763fc4baa6 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala @@ -39,6 +39,7 @@ package com.normation.rudder.repository import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ import com.normation.rudder.reports.execution.AgentRun @@ -72,7 +73,8 @@ trait ReportsRepository { * That method doesn't check if there is missing execution in * the result compared to inputs. */ - def getExecutionReports(runs: Set[AgentRunId], filterByRules: Set[RuleId]): Box[Map[NodeId, Seq[Reports]]] + def getExecutionReports(runs: Set[AgentRunId], filterByRules: Set[RuleId], filterByDirectives: Set[DirectiveId]): Box[Map[NodeId, Seq[Reports]]] + /** * Returns all reports for the node, between the two differents date (optionnal) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala index 011715d9716..5194e6c3833 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala @@ -206,6 +206,26 @@ class FindExpectedReportsJdbcRepository( } } + /** + * Return node ids associated to the directive (based on expectedreports (the one still pending)) for this Directive, + * only limited on the nodeIds in parameter (used when cache is incomplete) + */ + override def findCurrentNodeIdsForDirective(directiveId: DirectiveId, nodeIds: Set[NodeId]): IOResult[Set[NodeId]] = { + if (nodeIds.isEmpty) Set.empty[NodeId].succeed + else { + transactIOResult(s"Error when getting nodes for directive '${directiveId.serialize}' from expected reports")(xa => sql""" + select distinct nodeid from nodeconfigurations + where enddate is null and configuration like ${"%" + directiveId.serialize + "%"} + and nodeid in (${nodeIds.map(id => s"'${id}'").mkString(",")}) + """.query[NodeId].to[Set].transact(xa)) + } + } + override def findCurrentNodeIdsForDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] = { + transactIOResult(s"Error when getting nodes for directive '${directiveId.serialize}' from expected reports")(xa => sql""" + select distinct nodeid from nodeconfigurations + where enddate is null and configuration like ${"%" + directiveId.serialize + "%"} + """.query[NodeId].to[Set].transact(xa)) + } /* * Retrieve the list of node config ids */ diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala index b289aecf2f6..6969314f9ec 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala @@ -42,6 +42,7 @@ import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId import com.normation.rudder.db.Doobie import com.normation.rudder.db.Doobie._ +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ import com.normation.rudder.domain.reports.Reports @@ -50,6 +51,7 @@ import com.normation.rudder.reports.execution.AgentRunId import com.normation.rudder.repository.ReportsRepository import doobie._ import doobie.implicits._ + import java.sql.Timestamp import net.liftweb.common._ import org.joda.time._ @@ -77,25 +79,18 @@ class ReportsJdbcRepository(doobie: Doobie) extends ReportsRepository with Logga private[this] val idQuery = รพ(s"select id, ${common_reports_column} from ruddersysevents where 1=1 ") // We assume that this method is called with a limited list of runs - override def getExecutionReports(runs: Set[AgentRunId], filterByRules: Set[RuleId]): Box[Map[NodeId, Seq[Reports]]] = { - if (runs.isEmpty) Full(Map()) - else { - val ruleClause = { - if (filterByRules.isEmpty) "" - else s"and ruleid in ${filterByRules.map(_.serialize).mkString("('", "','", "')")}" - } - - val nodeParam = runs.map(x => s"('${x.nodeId.value}','${new Timestamp(x.date.getMillis)}'::timestamp)").mkString(",") - /* - * be careful in the number of parenthesis for "in values", it is: - * ... in (VALUES ('a', 'b') ); - * ... in (VALUES ('a', 'b'), ('c', 'd') ); - * etc. No more, no less. - */ - transactRunBox(xa => query[Reports](s"""select ${common_reports_column} - from RudderSysEvents - where (nodeid, executiontimestamp) in (VALUES ${nodeParam}) - """ + ruleClause).to[Vector].transact(xa)).map(_.groupBy(_.nodeId)) ?~! + override def getExecutionReports(runs: Set[AgentRunId], filterByRules: Set[RuleId], filterByDirectives: Set[DirectiveId]): Box[Map[NodeId, Seq[Reports]]] = { + runs.map(n => (n.nodeId,n.date)).toList.toNel match { + case None => Full(Map()) + case Some (nodeValues) => + + val ruleClause = filterByRules.toList.toNel.map(r => Fragments.in(fr"ruleid", r)) + val directiveClause = filterByDirectives.toList.toNel.map(r => Fragments.in(fr"directiveid", r)) + val values = Fragments.in(fr"(nodeid, executiontimestamp)",nodeValues) + val where = Fragments.whereAndOpt(Some(values),ruleClause,directiveClause) + + val q = sql"select ${common_reports_column} from RudderSysEvents" ++ where + transactRunBox(xa => q.query[Reports].to[Vector].transact(xa)).map(_.groupBy(_.nodeId)) ?~! s"Error when trying to get last run reports for ${runs.size} nodes" } } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala index 2360009cfe7..1d3812e3ede 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala @@ -40,6 +40,7 @@ package com.normation.rudder.services.reports import com.normation.errors._ import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.logger.ReportLoggerPure +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports.NodeAndConfigId import com.normation.rudder.domain.reports.NodeExpectedReports @@ -79,6 +80,7 @@ trait NodeConfigurationService { * */ def findNodesApplyingRule(ruleId: RuleId): IOResult[Set[NodeId]] + def findNodesApplyingDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] } trait NewExpectedReportsAvailableHook { @@ -281,6 +283,30 @@ class CachedNodeConfigurationService( } } + /** + * get the nodes applying the rule + * + */ + def findNodesApplyingDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] = { + // this don't need to be in a semaphore, since it's only one atomic cache read + for { + nodeConfs <- cache.get + nodesNotInCache = nodeConfs.collect { case (k, value) if (value.isEmpty) => k }.toSet + dataFromCache = { + nodeConfs.collect { + case (k, Some(nodeExpectedReports)) if (nodeExpectedReports.ruleExpectedReports.flatMap(_.directives.map(_.directiveId)).contains(directiveId)) => k + }.toSet + } + fromRepo <- if (nodesNotInCache.isEmpty) { + Set.empty[NodeId].succeed + } else { // query the repo + confExpectedRepo.findCurrentNodeIdsForDirective(directiveId, nodesNotInCache) + } + } yield { + dataFromCache ++ fromRepo + } + } + /** * retrieve expected reports by config version */ @@ -346,4 +372,12 @@ class NodeConfigurationServiceImpl( def findNodesApplyingRule(ruleId: RuleId): IOResult[Set[NodeId]] = { confExpectedRepo.findCurrentNodeIds(ruleId).toIO } + /** + * get the nodes applying the rule + * + */ + def findNodesApplyingDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] = { + confExpectedRepo.findCurrentNodeIdsForDirective(directiveId) + } + } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala index 54b306836e7..c303bf0d240 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala @@ -40,6 +40,7 @@ package com.normation.rudder.services.reports import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.logger.TimingDebugLogger +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports.ComplianceLevel import com.normation.rudder.domain.reports.NodeStatusReport @@ -57,6 +58,7 @@ trait ReportingService { * find node status reports for all rules) */ def findRuleNodeStatusReports(nodeIds: Set[NodeId], filterByRules: Set[RuleId]): Box[Map[NodeId, NodeStatusReport]] + def findDirectiveNodeStatusReports(nodeIds: Set[NodeId], filterByDirectives: Set[DirectiveId]): Box[Map[NodeId, NodeStatusReport]] def findUncomputedNodeStatusReports(): Box[Map[NodeId, NodeStatusReport]] @@ -103,6 +105,8 @@ trait ReportingService { */ def getUserNodeStatusReports(): Box[Map[NodeId, NodeStatusReport]] + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] + /** * find node status reports for user and system rules but in a separated couple (system is first element, user second) */ @@ -145,6 +149,22 @@ trait ReportingService { }.toMap } + def filterReportsByDirectives(reports: Map[NodeId, NodeStatusReport],directiveIds: Set[DirectiveId]): Map[NodeId, NodeStatusReport] = { + if (directiveIds.isEmpty) { + reports + } else + { + val n1 = System.currentTimeMillis + val result = reports.view.mapValues { + case status => + NodeStatusReport.filterByDirectives(status, directiveIds) + }.filter { case (_, v) => v.reports.nonEmpty || v.overrides.nonEmpty } + val n2 = System.currentTimeMillis + TimingDebugLogger.trace(s"Filter Node Status Reports on ${directiveIds.size} Directives in : ${n2 - n1}ms") + result + }.toMap + } + def complianceByRules(report: NodeStatusReport, ruleIds: Set[RuleId]): ComplianceLevel = { if (ruleIds.isEmpty) { report.compliance @@ -158,4 +178,18 @@ trait ReportingService { }) } } + + def complianceByRulesByDirectives(report: NodeStatusReport, ruleIds: Set[RuleId], directiveIds : Set[DirectiveId]): ComplianceLevel = { + if (ruleIds.isEmpty) { + report.compliance + } else { + // compute compliance only for the selected rules + // BE CAREFUL: reports is a SET - and it's likely that + // some compliance will be equals. So change to seq. + ComplianceLevel.sum(report.reports.toSeq.collect { + case report if (ruleIds.contains(report.ruleId)) => + report.compliance + }) + } + } } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala index 5d977a5ff76..b9d0fd2bc8f 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala @@ -45,6 +45,7 @@ import com.normation.rudder.domain.logger.ReportLoggerPure import com.normation.rudder.domain.logger.TimingDebugLogger import com.normation.rudder.domain.logger.TimingDebugLoggerPure import com.normation.rudder.domain.nodes.NodeState +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.GlobalPolicyMode import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ @@ -186,6 +187,24 @@ trait RuleOrNodeReportingServiceImpl extends ReportingService { } } + override def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = { + // here, the logic is ONLY to get the node for which that rule applies and then step back + // on the other method + for { + time_0 <- currentTimeMillis + nodeIds <- nodeConfigService.findNodesApplyingDirective(directiveId) + rules <- rulesRepo.getAll() + time_1 <- currentTimeMillis + _ <- TimingDebugLoggerPure.debug( + s"findCurrentNodeIds: Getting node IDs for directive '${directiveId.serialize}' took ${time_1 - time_0}ms" + ) + reports <- findDirectiveNodeStatusReports(nodeIds, Set(directiveId)).toIO + } yield { + reports + } + } + + override def findNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = { for { reports <- findRuleNodeStatusReports(Set(nodeId), Set()) @@ -603,6 +622,24 @@ trait CachedFindRuleNodeStatusReports } } + + /** + * Find node status reports. That method returns immediatly with the information it has in cache, which + * can be outdated. This is the prefered way to avoid huge contention (see https://issues.rudder.io/issues/16557). + * + * That method nonetheless check for expiration dates. + */ + override def findDirectiveNodeStatusReports(nodeIds: Set[NodeId], directiveIds: Set[DirectiveId]): Box[Map[NodeId, NodeStatusReport]] = { + val n1 = System.currentTimeMillis + for { + reports <- checkAndGetCache(nodeIds).toBox + n2 = System.currentTimeMillis + _ = ReportLogger.Cache.debug(s"Get node compliance from cache in: ${n2 - n1}ms") + } yield { + filterReportsByDirectives(reports, directiveIds) + } + } + /** * Retrieve a set of rule/node compliances given the nodes Id. * Optionally restrict the set to some rules if filterByRules is non empty (else, @@ -655,6 +692,9 @@ trait CachedFindRuleNodeStatusReports } } + + //def findStatusReportsForDirective(directive: DirectiveId): IOResult[NodeStatusReport] = + /** * Clear cache. Try a reload asynchronously, disregarding * the result @@ -722,7 +762,56 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ = TimingDebugLogger.trace(s"Compliance: get node run infos: ${t1 - t0}ms") // compute the status - nodeStatusReports <- buildNodeStatusReports(runInfos, ruleIds, complianceMode.mode, unexpectedMode) + nodeStatusReports <- buildNodeStatusReports(runInfos, ruleIds, Set(), complianceMode.mode, unexpectedMode) + + t2 = System.currentTimeMillis + _ = TimingDebugLogger.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") + } yield { + nodeStatusReports + } + } + + override def findDirectiveNodeStatusReports(nodeIds: Set[NodeId], directiveIds: Set[DirectiveId]): Box[Map[NodeId, NodeStatusReport]] = { + /* + * This is the main logic point to get reports. + * + * Compliance for a given node is a function of ONLY(expectedNodeConfigId, lastReceivedAgentRun). + * + * The logic is: + * + * - for a (or n) given node (we have a node-bias), + * - get the expected configuration right now + * - errors may happen if the node does not exist or if + * it does not have config right now. For example, it + * was added just a second ago. + * => "no data for that node" + * - get the last run for the node. + * + * If nodeConfigId(last run) == nodeConfigId(expected config) + * => simple compare & merge + * else { + * - expected reports INTERSECTION received report ==> compute the compliance on + * received reports (with an expiration date) + * - expected reports - received report ==> pending reports (with an expiration date) + * + * } + * + * All nodeIds get a value in the returnedMap, because: + * - getNodeRunInfos(nodeIds).keySet == nodeIds AND + * - runInfos.keySet == buildNodeStatusReports(runInfos,...).keySet + * So nodeIds === returnedMap.keySet holds + */ + val t0 = System.currentTimeMillis + for { + complianceMode <- getGlobalComplianceMode() + unexpectedMode <- getUnexpectedInterpretation() + // we want compliance on these nodes + runInfos <- getNodeRunInfos(nodeIds, complianceMode).toBox + t1 = System.currentTimeMillis + _ = TimingDebugLogger.trace(s"Compliance: get node run infos: ${t1 - t0}ms") + + // compute the status + nodeStatusReports <- buildNodeStatusReports(runInfos, Set(), directiveIds, complianceMode.mode, unexpectedMode) t2 = System.currentTimeMillis _ = TimingDebugLogger.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") @@ -745,7 +834,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ <- TimingDebugLoggerPure.trace(s"Compliance: get node run infos: ${t1 - t0}ms") // compute the status - nodeStatusReports <- buildNodeStatusReports(runInfos, filterByRules, complianceMode.mode, unexpectedMode).toIO + nodeStatusReports <- buildNodeStatusReports(runInfos, filterByRules, Set(), complianceMode.mode, unexpectedMode).toIO compliance = nodeStatusReports.map { case (k, v) => (k, v.compliance) } t2 <- currentTimeMillis _ <- TimingDebugLoggerPure.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") @@ -769,8 +858,8 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ = TimingDebugLogger.trace(s"Compliance: get node run infos: ${t1 - t0}ms") // compute the status - nodeUserStatusReports <- buildNodeStatusReports(runInfos, filterByUserRules, complianceMode.mode, unexpectedMode).toIO - nodeSystemStatusReports <- buildNodeStatusReports(runInfos, filterBySystemRules, complianceMode.mode, unexpectedMode).toIO + nodeUserStatusReports <- buildNodeStatusReports(runInfos, filterByUserRules, Set(), complianceMode.mode, unexpectedMode).toIO + nodeSystemStatusReports <- buildNodeStatusReports(runInfos, filterBySystemRules, Set(), complianceMode.mode, unexpectedMode).toIO nodeUserCompliance = nodeUserStatusReports.map { case (nodeId, nodeStatusReports) => (nodeId, nodeStatusReports.compliance) } @@ -855,7 +944,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ = TimingDebugLogger.trace(s"Compliance: get uncomputed node run infos: ${t1 - t0}ms") // compute the status - nodeStatusReports <- buildNodeStatusReports(uncomputedRuns, Set(), complianceMode.mode, unexpectedMode) + nodeStatusReports <- buildNodeStatusReports(uncomputedRuns, Set(), Set(), complianceMode.mode, unexpectedMode) t2 = System.currentTimeMillis _ = TimingDebugLogger.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") @@ -894,6 +983,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { private[this] def buildNodeStatusReports( runInfos: Map[NodeId, RunAndConfigInfo], ruleIds: Set[RuleId], + directiveIds: Set[DirectiveId], complianceModeName: ComplianceModeName, unexpectedInterpretation: UnexpectedReportInterpretation ): Box[Map[NodeId, NodeStatusReport]] = { @@ -926,7 +1016,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { */ reports <- complianceModeName match { case ReportsDisabled => Full(Map[NodeId, Seq[Reports]]()) - case _ => reportsRepository.getExecutionReports(agentRunIds, ruleIds) + case _ => reportsRepository.getExecutionReports(agentRunIds, ruleIds, directiveIds) } t1 = System.nanoTime() _ = u1 += (t1 - t0) diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala index 17e48c90d0d..caebd6a0fa4 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala @@ -167,6 +167,7 @@ class ReportingServiceTest extends DBCommon with BoxSpecMatcher { def nodeConfigrationService: NodeConfigurationService = null def findDirectiveRuleStatusReportsByRule(ruleId: RuleId): IOResult[Map[NodeId, NodeStatusReport]] = null def findNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = null def findUserNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null def findSystemNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null def getGlobalUserCompliance(): Box[Option[(ComplianceLevel, Long)]] = null @@ -179,6 +180,7 @@ class ReportingServiceTest extends DBCommon with BoxSpecMatcher { def computeComplianceFromReports(reports: Map[NodeId, NodeStatusReport]): Option[(ComplianceLevel, Long)] = None override def batchSize: Int = 5000 + } val RUDDER_JDBC_BATCH_MAX_SIZE = 5000 diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala index 56ca93c83d4..90c2db1c6a6 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala @@ -126,19 +126,19 @@ class ReportsTest extends DBCommon { } "find the last reports for node0" in { - val result = repostsRepo.getExecutionReports(Set(AgentRunId(NodeId("n0"), run1)), Set()).open + val result = repostsRepo.getExecutionReports(Set(AgentRunId(NodeId("n0"), run1)), Set(), Set()).open result.values.flatten.toSeq must contain(exactly(reports("n0")(0))) } "find reports for node 0,1,2" in { val runs = Set(("n0", run1), ("n1", run1), ("n2", run1)) - val result = repostsRepo.getExecutionReports(runs, Set()).open + val result = repostsRepo.getExecutionReports(runs, Set(), Set()).open result.values.flatten.toSeq must contain(exactly(reports("n0") ++ reports("n1").reverse.tail ++ reports("n2"): _*)) } "not find report for none existing agent run id" in { val runs = Set(("n2", run2), ("n3", run1)) - val result = repostsRepo.getExecutionReports(runs, Set()).open + val result = repostsRepo.getExecutionReports(runs, Set(), Set()).open result must beEmpty } } diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala index d278d092424..0a9d94634ff 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala @@ -41,6 +41,7 @@ import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.nodes.Node import com.normation.rudder.domain.nodes.NodeInfo +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports.ComplianceLevel import com.normation.rudder.domain.reports.NodeConfigId @@ -178,6 +179,8 @@ class CachedFindRuleNodeStatusReportsTest extends Specification { updated = (updated ++ nodeIds) Full(reports.filter(x => nodeIds.contains(x._1))) } + + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = ??? } override def nodeInfoService: NodeInfoService = testNodeInfoService @@ -193,6 +196,7 @@ class CachedFindRuleNodeStatusReportsTest extends Specification { optNodeIds: Option[Set[NodeId]] ): IOResult[(Map[NodeId, ComplianceLevel], Map[NodeId, ComplianceLevel])] = ??? + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = ??? } /* diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala index 7d1e6e5354d..8f53915ccf9 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala @@ -43,6 +43,7 @@ import com.normation.inventory.domain.NodeId import com.normation.rudder.api.ApiVersion import com.normation.rudder.domain.logger.TimingDebugLogger import com.normation.rudder.domain.logger.TimingDebugLoggerPure +import com.normation.rudder.domain.nodes.NodeInfo import com.normation.rudder.domain.policies.Directive import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.Rule @@ -72,6 +73,7 @@ import net.liftweb.http.PlainTextResponse import net.liftweb.http.Req import net.liftweb.json._ import net.liftweb.json.JsonDSL._ + import scala.collection.immutable import zio.syntax._ @@ -213,9 +215,10 @@ class ComplianceApi( precision <- restExtractor.extractPercentPrecision(req.params) format <- restExtractorService.extractComplianceFormat(req.params) id <- DirectiveId.parse(directiveId).toBox + d <- readDirective.getDirective(id.uid).notOptional(s"Directive with id '${id.serialize}' not found'").toBox t2 = System.currentTimeMillis _ = TimingDebugLogger.trace(s"API DirectiveCompliance - getting query param in ${t2 - t1} ms") - directive <- complianceService.getDirectiveCompliance(id, level) + directive <- complianceService.getDirectiveCompliance(d, level) t3 = System.currentTimeMillis _ = TimingDebugLogger.trace(s"API DirectiveCompliance - getting directive compliance '${id.uid.value}' in ${t3 - t2} ms") } yield { @@ -259,12 +262,8 @@ class ComplianceApi( precision <- restExtractor.extractPercentPrecision(req.params) t2 = System.currentTimeMillis _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting query param in ${t2 - t1} ms") - fullLibrary <- readDirective.getFullDirectiveLibrary().toBox ?~! "Could not fetch Directives" - directiveIds = fullLibrary.allDirectives.values.filter(!_._2.isSystem).map(_._2.id).toList - t3 = System.currentTimeMillis - _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting directives id ${t3 - t2} ms") t4 = System.currentTimeMillis - directives = directiveIds.flatMap(complianceService.getDirectiveCompliance(_, level)) + directives <- complianceService.getDirectivesCompliance(level) t5 = System.currentTimeMillis _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting directives compliance in ${t5 - t4} ms") @@ -389,6 +388,107 @@ class ComplianceAPIService( val getGlobalComplianceMode: () => Box[GlobalComplianceMode] ) { + + private[this] def components(nodeInfos : Map[NodeId, NodeInfo])(name: String, nodeComponents: List[(NodeId, ComponentStatusReport)]): List[ByRuleComponentCompliance] = { + + val (groupsComponents, uniqueComponents) = nodeComponents.partitionMap { + case (a, b: BlockStatusReport) => Left((a, b)) + case (a, b: ValueStatusReport) => Right((a, b)) + } + + (if (groupsComponents.isEmpty) { + Nil + } else { + val bidule = groupsComponents.flatMap { case (nodeId, c) => c.subComponents.map(sub => (nodeId, sub)) } + .groupBy((_._2.componentName)) + ByRuleBlockCompliance( + name, + ComplianceLevel.sum(groupsComponents.map(_._2.compliance)), + bidule.flatMap(c => components(nodeInfos)(c._1, c._2)).toList + ) :: Nil + }) ::: (if (uniqueComponents.isEmpty) { + Nil + } else { + ByRuleValueCompliance( + name, + ComplianceLevel.sum( + uniqueComponents.map(_._2.compliance) + ), // here, we finally group by nodes for each components ! + { + val byNode = uniqueComponents.groupBy(_._1) + byNode.map { + case (nodeId, components) => + ByRuleNodeCompliance( + nodeId, + nodeInfos.get(nodeId).map(_.hostname).getOrElse("Unknown node"), + ComplianceLevel.sum(components.map(_._2.compliance)), + components.sortBy(_._2.componentName).flatMap(_._2.componentValues) + ) + }.toSeq + } + ) :: Nil + }) + } + + private[this] def getByDirectivesCompliance(directives: Seq[Directive], level: Option[Int]): IOResult[Seq[ByDirectiveCompliance]] = { + val computedLevel = level.getOrElse(10) + + for { + t1 <- currentTimeMillis + allGroups <- nodeGroupRepo.getAllNodeIds() + t2 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - nodeGroupRepo.getAllNodeIds in ${t2 - t1} ms") + + // this can be optimized, as directive only happen for level=2 + rules <- if (computedLevel >= 2) { + rulesRepo.getAll() + } else { + Seq().succeed + } + t3 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - getFullDirectiveLibrary in ${t3 - t2} ms") + + nodeInfos <- nodeInfoService.getAll() + t4 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - nodeInfoService.getAll() in ${t4 - t3} ms") + + compliance <- getGlobalComplianceMode().toIO + t5 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - getGlobalComplianceMode in ${t5 - t4} ms") + reportsByNode <- reportingService + .findDirectiveNodeStatusReports( + nodeInfos.keySet, + directives.map(_.id).toSet + ).toIO + t6 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - findRuleNodeStatusReports in ${t6 - t5} ms") + + } yield { + + val reportsByRule = reportsByNode.flatMap { case (_, status) => status.reports }.groupBy(_.ruleId) + val t7 = System.currentTimeMillis() + TimingDebugLoggerPure.logEffect.trace(s"getByRulesCompliance - group reports by rules in ${t7 - t6} ms") + for { + directive <- directives + } yield { + val rulesCompliance = for { + (ruleId, reportsByDir) <- reportsByRule.toSeq.map { r => + val g = r._2.map(r =>(r.nodeId,r)) + val t = g.flatMap(n => n._2.directives.get(directive.id).toList.flatMap(_.components.map((n._1,_)))).groupBy(_._2.componentName) + (r._1,t.toSeq) + } + } yield { + val c = reportsByDir.flatMap(c => components(nodeInfos)(c._1,c._2.toList)) + + val ruleName = rules.find(_.id == ruleId).map(_.name).getOrElse("") + ByDirectiveByRuleCompliance(ruleId,ruleName, ComplianceLevel.sum(c.map(_.compliance)),c) + } + //level = ComplianceLevel.sum(reportsByDir.map(_.compliance)) + ByDirectiveCompliance(directive.id, directive.name, ComplianceLevel.sum(rulesCompliance.map(_.compliance)), compliance.mode, rulesCompliance) + } + } + } + /** * Get the compliance for everything * level is optionnally the selected level. @@ -450,47 +550,6 @@ class ComplianceAPIService( reports.flatMap(r => r.directives.values.map(d => (r.nodeId, d)).toSeq).groupBy(_._2.directiveId) } - def components(name: String, nodeComponents: List[(NodeId, ComponentStatusReport)]): List[ByRuleComponentCompliance] = { - - val (groupsComponents, uniqueComponents) = nodeComponents.partitionMap { - case (a, b: BlockStatusReport) => Left((a, b)) - case (a, b: ValueStatusReport) => Right((a, b)) - } - - (if (groupsComponents.isEmpty) { - Nil - } else { - val bidule = groupsComponents.flatMap { case (nodeId, c) => c.subComponents.map(sub => (nodeId, sub)) } - .groupBy((_._2.componentName)) - ByRuleBlockCompliance( - name, - ComplianceLevel.sum(groupsComponents.map(_._2.compliance)), - bidule.flatMap(c => components(c._1, c._2)).toList - ) :: Nil - }) ::: (if (uniqueComponents.isEmpty) { - Nil - } else { - ByRuleValueCompliance( - name, - ComplianceLevel.sum( - uniqueComponents.map(_._2.compliance) - ), // here, we finally group by nodes for each components ! - { - val byNode = uniqueComponents.groupBy(_._1) - byNode.map { - case (nodeId, components) => - ByRuleNodeCompliance( - nodeId, - nodeInfos.get(nodeId).map(_.hostname).getOrElse("Unknown node"), - ComplianceLevel.sum(components.map(_._2.compliance)), - components.sortBy(_._2.componentName).flatMap(_._2.componentValues) - ) - }.toSeq - } - ) :: Nil - }) - } - ByRuleRuleCompliance( ruleId, ruleObjects.get(ruleId).map(_.name).getOrElse("Unknown rule"), @@ -512,7 +571,7 @@ class ComplianceAPIService( nodeDirectives.flatMap { case (nodeId, d) => d.components.map(c => (nodeId, c)).toSeq } .groupBy(_._2.componentName) } - byComponents.flatMap { case (name, nodeComponents) => components(name, nodeComponents.toList) }.toSeq + byComponents.flatMap { case (name, nodeComponents) => components(nodeInfos)(name, nodeComponents.toList) }.toSeq } ) }.toSeq @@ -569,34 +628,17 @@ class ComplianceAPIService( } }.toBox - def getDirectiveCompliance(directiveId: DirectiveId, level: Option[Int]): Box[ByDirectiveCompliance] = { - for { - rules <- rulesRepo.getAll() - directive <- directiveRepo.getDirective(directiveId.uid) - relevantRules = rules.filter(_.directiveIds.contains(directiveId)) - - byRules <- getByRulesCompliance(relevantRules, level) - rules = byRules.flatMap { rule => - rule.directives.map { directive => - ByDirectiveByRuleCompliance( - rule.id, - rule.name, - rule.compliance, - directive.components - ) - } - } + def getDirectiveCompliance(directive: Directive, level: Option[Int]): Box[ByDirectiveCompliance] = { + getByDirectivesCompliance(Seq(directive),level).flatMap(_.find(_.id == directive.id).notOptional(s"No reports were found for directive with ID '${directive.id.serialize}'")).toBox + } - compliance <- getGlobalComplianceMode().toIO + def getDirectivesCompliance(level: Option[Int]): Box[Seq[ByDirectiveCompliance]] = { + for { + directives <- directiveRepo.getFullDirectiveLibrary().map(_.allDirectives.values.map(_._2)) + reports <- getByDirectivesCompliance(directives.toSeq, level) } yield { - ByDirectiveCompliance( - directiveId, - directive.map(_.name).getOrElse("Unknown"), - ComplianceLevel.sum(byRules.map(_.compliance)), - compliance.mode, - rules - ) + reports } }.toBox @@ -668,10 +710,10 @@ class ComplianceAPIService( ) }.toSeq ) - }).toSeq + }) ) ) - }.toMap + } } // for each rule for each node, we want to have a