From 2efc650b7d21db8d527d0d994e95f0e5252ee773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Membr=C3=A9?= Date: Tue, 28 Mar 2023 20:25:41 +0200 Subject: [PATCH] Fixes #22552: General improvements on Directive compliance API --- .../com/normation/rudder/db/Doobie.scala | 16 +- .../rudder/domain/reports/StatusReports.scala | 14 ++ .../ExpectedReportsRepository.scala | 8 + .../rudder/repository/ReportsRepository.scala | 7 +- .../jdbc/ExpectedReportsJdbcRepository.scala | 22 +- .../jdbc/ReportsJdbcRepository.scala | 36 ++- .../reports/NodeConfigurationService.scala | 39 ++- .../services/reports/ReportingService.scala | 44 ++++ .../reports/ReportingServiceImpl.scala | 107 +++++++- .../jdbc/ReportingServiceTest.scala | 20 +- .../rudder/repository/jdbc/ReportsTest.scala | 6 +- .../CachedFindRuleNodeStatusReportsTest.scala | 4 + .../rudder/rest/lift/ComplianceApi.scala | 233 +++++++++++------- 13 files changed, 423 insertions(+), 133 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala index 7695e7e5bc8..76c1d36a202 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/db/Doobie.scala @@ -170,8 +170,8 @@ object Doobie { implicit val DateTimeMeta: Meta[DateTime] = Meta[java.sql.Timestamp].imap(ts => new DateTime(ts.getTime()))(dt => new java.sql.Timestamp(dt.getMillis)) - implicit val ReadRuleId: Read[RuleId] = { - Read[String].map(r => { + implicit val ReadRuleId: Get[RuleId] = { + Get[String].map(r => { RuleId.parse(r) match { case Right(rid) => rid case Left(err) => @@ -179,11 +179,11 @@ object Doobie { } }) } - implicit val WriteRuleId: Write[RuleId] = { - Write[String].contramap(_.serialize) + implicit val PutRuleId: Put[RuleId] = { + Put[String].contramap(_.serialize) } - implicit val ReadDirectiveId: Read[DirectiveId] = { - Read[String].map(r => { + implicit val ReadDirectiveId: Get[DirectiveId] = { + Get[String].map(r => { DirectiveId.parse(r) match { case Right(rid) => rid case Left(err) => @@ -193,8 +193,8 @@ object Doobie { } }) } - implicit val WriteDirectiveId: Write[DirectiveId] = { - Write[String].contramap(_.serialize) + implicit val WriteDirectiveId: Put[DirectiveId] = { + Put[String].contramap(_.serialize) } implicit val ReportRead: Read[Reports] = { diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala index 993e14f9bdd..8ec9dc7c387 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/reports/StatusReports.scala @@ -153,6 +153,20 @@ object NodeStatusReport { nodeStatusReport.reports.filter(r => ruleIds.contains(r.ruleId)) ) } + + def filterByDirectives(nodeStatusReport: NodeStatusReport, directiveIds: Set[DirectiveId]): NodeStatusReport = { + + new NodeStatusReport( + nodeStatusReport.nodeId, + nodeStatusReport.runInfo, + nodeStatusReport.statusInfo, + nodeStatusReport.overrides, + nodeStatusReport.reports.flatMap { r => + val filterRule = r.copy(directives = r.directives.filter(d => directiveIds.contains(d._1))) + if (filterRule.directives.isEmpty) None else Some(filterRule) + } + ) + } } /** diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala index 865df611234..e06b5aab3c0 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ExpectedReportsRepository.scala @@ -39,6 +39,7 @@ package com.normation.rudder.repository import com.normation.box._ import com.normation.errors._ import com.normation.inventory.domain.NodeId +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ import net.liftweb.common.Box @@ -109,6 +110,13 @@ trait FindExpectedReportRepository { */ def findCurrentNodeIdsForRule(ruleId: RuleId, nodeIds: Set[NodeId]): IOResult[Set[NodeId]] + /** + * Return node ids associated to the rule (based on expectedreports (the one still pending)) for this Rule, + * only limited on the nodeIds in parameter (used when cache is incomplete) + */ + def findCurrentNodeIdsForDirective(ruleId: DirectiveId, nodeIds: Set[NodeId]): IOResult[Set[NodeId]] + def findCurrentNodeIdsForDirective(ruleId: DirectiveId): IOResult[Set[NodeId]] + /* * Retrieve the expected reports by config version of the nodes. * diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala index 92f590c535c..e0fa23acc17 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ReportsRepository.scala @@ -39,6 +39,7 @@ package com.normation.rudder.repository import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ import com.normation.rudder.reports.execution.AgentRun @@ -72,7 +73,11 @@ trait ReportsRepository { * That method doesn't check if there is missing execution in * the result compared to inputs. */ - def getExecutionReports(runs: Set[AgentRunId], filterByRules: Set[RuleId]): Box[Map[NodeId, Seq[Reports]]] + def getExecutionReports( + runs: Set[AgentRunId], + filterByRules: Set[RuleId], + filterByDirectives: Set[DirectiveId] + ): Box[Map[NodeId, Seq[Reports]]] /** * Returns all reports for the node, between the two differents date (optionnal) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala index 011715d9716..7d071c586a5 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala @@ -206,10 +206,30 @@ class FindExpectedReportsJdbcRepository( } } + /** + * Return node ids associated to the directive (based on expectedreports (the one still pending)) for this Directive, + * only limited on the nodeIds in parameter (used when cache is incomplete) + */ + override def findCurrentNodeIdsForDirective(directiveId: DirectiveId, nodeIds: Set[NodeId]): IOResult[Set[NodeId]] = { + if (nodeIds.isEmpty) Set.empty[NodeId].succeed + else { + transactIOResult(s"Error when getting nodes for directive '${directiveId.serialize}' from expected reports")(xa => sql""" + select distinct nodeid from nodeconfigurations + where enddate is null and configuration like ${"%" + directiveId.serialize + "%"} + and nodeid in (${nodeIds.map(id => s"'${id}'").mkString(",")}) + """.query[NodeId].to[Set].transact(xa)) + } + } + override def findCurrentNodeIdsForDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] = { + transactIOResult(s"Error when getting nodes for directive '${directiveId.serialize}' from expected reports")(xa => sql""" + select distinct nodeid from nodeconfigurations + where enddate is null and configuration like ${"%" + directiveId.serialize + "%"} + """.query[NodeId].to[Set].transact(xa)) + } /* * Retrieve the list of node config ids */ - override def getNodeConfigIdInfos(nodeIds: Set[NodeId]): Box[Map[NodeId, Option[Vector[NodeConfigIdInfo]]]] = { + override def getNodeConfigIdInfos(nodeIds: Set[NodeId]): Box[Map[NodeId, Option[Vector[NodeConfigIdInfo]]]] = { if (nodeIds.isEmpty) Full(Map.empty[NodeId, Option[Vector[NodeConfigIdInfo]]]) else { val batchedNodesId = nodeIds.grouped(jdbcMaxBatchSize).toSeq diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala index b289aecf2f6..b38678c5b96 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/jdbc/ReportsJdbcRepository.scala @@ -42,6 +42,7 @@ import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId import com.normation.rudder.db.Doobie import com.normation.rudder.db.Doobie._ +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ import com.normation.rudder.domain.reports.Reports @@ -77,26 +78,23 @@ class ReportsJdbcRepository(doobie: Doobie) extends ReportsRepository with Logga private[this] val idQuery = รพ(s"select id, ${common_reports_column} from ruddersysevents where 1=1 ") // We assume that this method is called with a limited list of runs - override def getExecutionReports(runs: Set[AgentRunId], filterByRules: Set[RuleId]): Box[Map[NodeId, Seq[Reports]]] = { - if (runs.isEmpty) Full(Map()) - else { - val ruleClause = { - if (filterByRules.isEmpty) "" - else s"and ruleid in ${filterByRules.map(_.serialize).mkString("('", "','", "')")}" - } + override def getExecutionReports( + runs: Set[AgentRunId], + filterByRules: Set[RuleId], + filterByDirectives: Set[DirectiveId] + ): Box[Map[NodeId, Seq[Reports]]] = { + runs.map(n => (n.nodeId.value, n.date)).toList.toNel match { + case None => Full(Map()) + case Some(nodeValues) => + val ruleClause = filterByRules.toList.toNel.map(r => Fragments.in(fr"ruleid", r)) + val directiveClause = filterByDirectives.toList.toNel.map(r => Fragments.in(fr"directiveid", r)) + val values = Fragments.in(fr"(nodeid, executiontimestamp)", nodeValues) + val where = Fragments.whereAndOpt(Some(values), ruleClause, directiveClause) - val nodeParam = runs.map(x => s"('${x.nodeId.value}','${new Timestamp(x.date.getMillis)}'::timestamp)").mkString(",") - /* - * be careful in the number of parenthesis for "in values", it is: - * ... in (VALUES ('a', 'b') ); - * ... in (VALUES ('a', 'b'), ('c', 'd') ); - * etc. No more, no less. - */ - transactRunBox(xa => query[Reports](s"""select ${common_reports_column} - from RudderSysEvents - where (nodeid, executiontimestamp) in (VALUES ${nodeParam}) - """ + ruleClause).to[Vector].transact(xa)).map(_.groupBy(_.nodeId)) ?~! - s"Error when trying to get last run reports for ${runs.size} nodes" + val q = + sql"select executiondate, ruleid, directiveid, nodeid, reportid, component, keyvalue, executiontimestamp, eventtype, msg from RudderSysEvents " ++ where + transactRunBox(xa => q.query[Reports].to[Vector].transact(xa)).map(_.groupBy(_.nodeId)) ?~! + s"Error when trying to get last run reports for ${runs.size} nodes" } } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala index 2360009cfe7..fb19a09ba47 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/NodeConfigurationService.scala @@ -40,6 +40,7 @@ package com.normation.rudder.services.reports import com.normation.errors._ import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.logger.ReportLoggerPure +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports.NodeAndConfigId import com.normation.rudder.domain.reports.NodeExpectedReports @@ -78,7 +79,8 @@ trait NodeConfigurationService { * get the nodes applying the rule * */ - def findNodesApplyingRule(ruleId: RuleId): IOResult[Set[NodeId]] + def findNodesApplyingRule(ruleId: RuleId): IOResult[Set[NodeId]] + def findNodesApplyingDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] } trait NewExpectedReportsAvailableHook { @@ -281,6 +283,32 @@ class CachedNodeConfigurationService( } } + /** + * get the nodes applying the rule + * + */ + def findNodesApplyingDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] = { + // this don't need to be in a semaphore, since it's only one atomic cache read + for { + nodeConfs <- cache.get + nodesNotInCache = nodeConfs.collect { case (k, value) if (value.isEmpty) => k }.toSet + dataFromCache = { + nodeConfs.collect { + case (k, Some(nodeExpectedReports)) + if (nodeExpectedReports.ruleExpectedReports.flatMap(_.directives.map(_.directiveId)).contains(directiveId)) => + k + }.toSet + } + fromRepo <- if (nodesNotInCache.isEmpty) { + Set.empty[NodeId].succeed + } else { // query the repo + confExpectedRepo.findCurrentNodeIdsForDirective(directiveId, nodesNotInCache) + } + } yield { + dataFromCache ++ fromRepo + } + } + /** * retrieve expected reports by config version */ @@ -346,4 +374,13 @@ class NodeConfigurationServiceImpl( def findNodesApplyingRule(ruleId: RuleId): IOResult[Set[NodeId]] = { confExpectedRepo.findCurrentNodeIds(ruleId).toIO } + + /** + * get the nodes applying the rule + * + */ + def findNodesApplyingDirective(directiveId: DirectiveId): IOResult[Set[NodeId]] = { + confExpectedRepo.findCurrentNodeIdsForDirective(directiveId) + } + } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala index 54b306836e7..55286799ae1 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingService.scala @@ -40,6 +40,7 @@ package com.normation.rudder.services.reports import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.logger.TimingDebugLogger +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports.ComplianceLevel import com.normation.rudder.domain.reports.NodeStatusReport @@ -57,6 +58,10 @@ trait ReportingService { * find node status reports for all rules) */ def findRuleNodeStatusReports(nodeIds: Set[NodeId], filterByRules: Set[RuleId]): Box[Map[NodeId, NodeStatusReport]] + def findDirectiveNodeStatusReports( + nodeIds: Set[NodeId], + filterByDirectives: Set[DirectiveId] + ): Box[Map[NodeId, NodeStatusReport]] def findUncomputedNodeStatusReports(): Box[Map[NodeId, NodeStatusReport]] @@ -103,6 +108,8 @@ trait ReportingService { */ def getUserNodeStatusReports(): Box[Map[NodeId, NodeStatusReport]] + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] + /** * find node status reports for user and system rules but in a separated couple (system is first element, user second) */ @@ -145,6 +152,25 @@ trait ReportingService { }.toMap } + def filterReportsByDirectives( + reports: Map[NodeId, NodeStatusReport], + directiveIds: Set[DirectiveId] + ): Map[NodeId, NodeStatusReport] = { + if (directiveIds.isEmpty) { + reports + } else + { + val n1 = System.currentTimeMillis + val result = reports.view.mapValues { + case status => + NodeStatusReport.filterByDirectives(status, directiveIds) + }.filter { case (_, v) => v.reports.nonEmpty || v.overrides.nonEmpty } + val n2 = System.currentTimeMillis + TimingDebugLogger.trace(s"Filter Node Status Reports on ${directiveIds.size} Directives in : ${n2 - n1}ms") + result + }.toMap + } + def complianceByRules(report: NodeStatusReport, ruleIds: Set[RuleId]): ComplianceLevel = { if (ruleIds.isEmpty) { report.compliance @@ -158,4 +184,22 @@ trait ReportingService { }) } } + + def complianceByRulesByDirectives( + report: NodeStatusReport, + ruleIds: Set[RuleId], + directiveIds: Set[DirectiveId] + ): ComplianceLevel = { + if (ruleIds.isEmpty) { + report.compliance + } else { + // compute compliance only for the selected rules + // BE CAREFUL: reports is a SET - and it's likely that + // some compliance will be equals. So change to seq. + ComplianceLevel.sum(report.reports.toSeq.collect { + case report if (ruleIds.contains(report.ruleId)) => + report.compliance + }) + } + } } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala index 5d977a5ff76..b218b7b91bc 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/reports/ReportingServiceImpl.scala @@ -45,6 +45,7 @@ import com.normation.rudder.domain.logger.ReportLoggerPure import com.normation.rudder.domain.logger.TimingDebugLogger import com.normation.rudder.domain.logger.TimingDebugLoggerPure import com.normation.rudder.domain.nodes.NodeState +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.GlobalPolicyMode import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports._ @@ -186,6 +187,23 @@ trait RuleOrNodeReportingServiceImpl extends ReportingService { } } + override def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = { + // here, the logic is ONLY to get the node for which that rule applies and then step back + // on the other method + for { + time_0 <- currentTimeMillis + nodeIds <- nodeConfigService.findNodesApplyingDirective(directiveId) + rules <- rulesRepo.getAll() + time_1 <- currentTimeMillis + _ <- TimingDebugLoggerPure.debug( + s"findCurrentNodeIds: Getting node IDs for directive '${directiveId.serialize}' took ${time_1 - time_0}ms" + ) + reports <- findDirectiveNodeStatusReports(nodeIds, Set(directiveId)).toIO + } yield { + reports + } + } + override def findNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = { for { reports <- findRuleNodeStatusReports(Set(nodeId), Set()) @@ -603,6 +621,26 @@ trait CachedFindRuleNodeStatusReports } } + /** + * Find node status reports. That method returns immediatly with the information it has in cache, which + * can be outdated. This is the prefered way to avoid huge contention (see https://issues.rudder.io/issues/16557). + * + * That method nonetheless check for expiration dates. + */ + override def findDirectiveNodeStatusReports( + nodeIds: Set[NodeId], + directiveIds: Set[DirectiveId] + ): Box[Map[NodeId, NodeStatusReport]] = { + val n1 = System.currentTimeMillis + for { + reports <- checkAndGetCache(nodeIds).toBox + n2 = System.currentTimeMillis + _ = ReportLogger.Cache.debug(s"Get node compliance from cache in: ${n2 - n1}ms") + } yield { + filterReportsByDirectives(reports, directiveIds) + } + } + /** * Retrieve a set of rule/node compliances given the nodes Id. * Optionally restrict the set to some rules if filterByRules is non empty (else, @@ -655,6 +693,8 @@ trait CachedFindRuleNodeStatusReports } } + // def findStatusReportsForDirective(directive: DirectiveId): IOResult[NodeStatusReport] = + /** * Clear cache. Try a reload asynchronously, disregarding * the result @@ -722,7 +762,59 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ = TimingDebugLogger.trace(s"Compliance: get node run infos: ${t1 - t0}ms") // compute the status - nodeStatusReports <- buildNodeStatusReports(runInfos, ruleIds, complianceMode.mode, unexpectedMode) + nodeStatusReports <- buildNodeStatusReports(runInfos, ruleIds, Set(), complianceMode.mode, unexpectedMode) + + t2 = System.currentTimeMillis + _ = TimingDebugLogger.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") + } yield { + nodeStatusReports + } + } + + override def findDirectiveNodeStatusReports( + nodeIds: Set[NodeId], + directiveIds: Set[DirectiveId] + ): Box[Map[NodeId, NodeStatusReport]] = { + /* + * This is the main logic point to get reports. + * + * Compliance for a given node is a function of ONLY(expectedNodeConfigId, lastReceivedAgentRun). + * + * The logic is: + * + * - for a (or n) given node (we have a node-bias), + * - get the expected configuration right now + * - errors may happen if the node does not exist or if + * it does not have config right now. For example, it + * was added just a second ago. + * => "no data for that node" + * - get the last run for the node. + * + * If nodeConfigId(last run) == nodeConfigId(expected config) + * => simple compare & merge + * else { + * - expected reports INTERSECTION received report ==> compute the compliance on + * received reports (with an expiration date) + * - expected reports - received report ==> pending reports (with an expiration date) + * + * } + * + * All nodeIds get a value in the returnedMap, because: + * - getNodeRunInfos(nodeIds).keySet == nodeIds AND + * - runInfos.keySet == buildNodeStatusReports(runInfos,...).keySet + * So nodeIds === returnedMap.keySet holds + */ + val t0 = System.currentTimeMillis + for { + complianceMode <- getGlobalComplianceMode() + unexpectedMode <- getUnexpectedInterpretation() + // we want compliance on these nodes + runInfos <- getNodeRunInfos(nodeIds, complianceMode).toBox + t1 = System.currentTimeMillis + _ = TimingDebugLogger.trace(s"Compliance: get node run infos: ${t1 - t0}ms") + + // compute the status + nodeStatusReports <- buildNodeStatusReports(runInfos, Set(), directiveIds, complianceMode.mode, unexpectedMode) t2 = System.currentTimeMillis _ = TimingDebugLogger.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") @@ -745,7 +837,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ <- TimingDebugLoggerPure.trace(s"Compliance: get node run infos: ${t1 - t0}ms") // compute the status - nodeStatusReports <- buildNodeStatusReports(runInfos, filterByRules, complianceMode.mode, unexpectedMode).toIO + nodeStatusReports <- buildNodeStatusReports(runInfos, filterByRules, Set(), complianceMode.mode, unexpectedMode).toIO compliance = nodeStatusReports.map { case (k, v) => (k, v.compliance) } t2 <- currentTimeMillis _ <- TimingDebugLoggerPure.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") @@ -769,8 +861,10 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ = TimingDebugLogger.trace(s"Compliance: get node run infos: ${t1 - t0}ms") // compute the status - nodeUserStatusReports <- buildNodeStatusReports(runInfos, filterByUserRules, complianceMode.mode, unexpectedMode).toIO - nodeSystemStatusReports <- buildNodeStatusReports(runInfos, filterBySystemRules, complianceMode.mode, unexpectedMode).toIO + nodeUserStatusReports <- + buildNodeStatusReports(runInfos, filterByUserRules, Set(), complianceMode.mode, unexpectedMode).toIO + nodeSystemStatusReports <- + buildNodeStatusReports(runInfos, filterBySystemRules, Set(), complianceMode.mode, unexpectedMode).toIO nodeUserCompliance = nodeUserStatusReports.map { case (nodeId, nodeStatusReports) => (nodeId, nodeStatusReports.compliance) } @@ -855,7 +949,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { _ = TimingDebugLogger.trace(s"Compliance: get uncomputed node run infos: ${t1 - t0}ms") // compute the status - nodeStatusReports <- buildNodeStatusReports(uncomputedRuns, Set(), complianceMode.mode, unexpectedMode) + nodeStatusReports <- buildNodeStatusReports(uncomputedRuns, Set(), Set(), complianceMode.mode, unexpectedMode) t2 = System.currentTimeMillis _ = TimingDebugLogger.debug(s"Compliance: compute compliance reports: ${t2 - t1}ms") @@ -894,6 +988,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { private[this] def buildNodeStatusReports( runInfos: Map[NodeId, RunAndConfigInfo], ruleIds: Set[RuleId], + directiveIds: Set[DirectiveId], complianceModeName: ComplianceModeName, unexpectedInterpretation: UnexpectedReportInterpretation ): Box[Map[NodeId, NodeStatusReport]] = { @@ -926,7 +1021,7 @@ trait DefaultFindRuleNodeStatusReports extends ReportingService { */ reports <- complianceModeName match { case ReportsDisabled => Full(Map[NodeId, Seq[Reports]]()) - case _ => reportsRepository.getExecutionReports(agentRunIds, ruleIds) + case _ => reportsRepository.getExecutionReports(agentRunIds, ruleIds, directiveIds) } t1 = System.nanoTime() _ = u1 += (t1 - t0) diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala index 17e48c90d0d..0ad3ab15079 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala @@ -162,15 +162,16 @@ class ReportingServiceTest extends DBCommon with BoxSpecMatcher { } val dummyComplianceCache = new CachedFindRuleNodeStatusReports { - def defaultFindRuleNodeStatusReports: DefaultFindRuleNodeStatusReports = null - def nodeInfoService: NodeInfoService = self.nodeInfoService - def nodeConfigrationService: NodeConfigurationService = null - def findDirectiveRuleStatusReportsByRule(ruleId: RuleId): IOResult[Map[NodeId, NodeStatusReport]] = null - def findNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null - def findUserNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null - def findSystemNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null - def getGlobalUserCompliance(): Box[Option[(ComplianceLevel, Long)]] = null - def findUncomputedNodeStatusReports(): Box[Map[NodeId, NodeStatusReport]] = null + def defaultFindRuleNodeStatusReports: DefaultFindRuleNodeStatusReports = null + def nodeInfoService: NodeInfoService = self.nodeInfoService + def nodeConfigrationService: NodeConfigurationService = null + def findDirectiveRuleStatusReportsByRule(ruleId: RuleId): IOResult[Map[NodeId, NodeStatusReport]] = null + def findNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = null + def findUserNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null + def findSystemNodeStatusReport(nodeId: NodeId): Box[NodeStatusReport] = null + def getGlobalUserCompliance(): Box[Option[(ComplianceLevel, Long)]] = null + def findUncomputedNodeStatusReports(): Box[Map[NodeId, NodeStatusReport]] = null def getUserNodeStatusReports(): Box[Map[NodeId, NodeStatusReport]] = Full(Map()) def getSystemAndUserCompliance( @@ -179,6 +180,7 @@ class ReportingServiceTest extends DBCommon with BoxSpecMatcher { def computeComplianceFromReports(reports: Map[NodeId, NodeStatusReport]): Option[(ComplianceLevel, Long)] = None override def batchSize: Int = 5000 + } val RUDDER_JDBC_BATCH_MAX_SIZE = 5000 diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala index 56ca93c83d4..90c2db1c6a6 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportsTest.scala @@ -126,19 +126,19 @@ class ReportsTest extends DBCommon { } "find the last reports for node0" in { - val result = repostsRepo.getExecutionReports(Set(AgentRunId(NodeId("n0"), run1)), Set()).open + val result = repostsRepo.getExecutionReports(Set(AgentRunId(NodeId("n0"), run1)), Set(), Set()).open result.values.flatten.toSeq must contain(exactly(reports("n0")(0))) } "find reports for node 0,1,2" in { val runs = Set(("n0", run1), ("n1", run1), ("n2", run1)) - val result = repostsRepo.getExecutionReports(runs, Set()).open + val result = repostsRepo.getExecutionReports(runs, Set(), Set()).open result.values.flatten.toSeq must contain(exactly(reports("n0") ++ reports("n1").reverse.tail ++ reports("n2"): _*)) } "not find report for none existing agent run id" in { val runs = Set(("n2", run2), ("n3", run1)) - val result = repostsRepo.getExecutionReports(runs, Set()).open + val result = repostsRepo.getExecutionReports(runs, Set(), Set()).open result must beEmpty } } diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala index d278d092424..0a9d94634ff 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala @@ -41,6 +41,7 @@ import com.normation.errors.IOResult import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.nodes.Node import com.normation.rudder.domain.nodes.NodeInfo +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.reports.ComplianceLevel import com.normation.rudder.domain.reports.NodeConfigId @@ -178,6 +179,8 @@ class CachedFindRuleNodeStatusReportsTest extends Specification { updated = (updated ++ nodeIds) Full(reports.filter(x => nodeIds.contains(x._1))) } + + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = ??? } override def nodeInfoService: NodeInfoService = testNodeInfoService @@ -193,6 +196,7 @@ class CachedFindRuleNodeStatusReportsTest extends Specification { optNodeIds: Option[Set[NodeId]] ): IOResult[(Map[NodeId, ComplianceLevel], Map[NodeId, ComplianceLevel])] = ??? + def findStatusReportsForDirective(directiveId: DirectiveId): IOResult[Map[NodeId, NodeStatusReport]] = ??? } /* diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala index 7d1e6e5354d..98992c0ecdc 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala @@ -43,6 +43,7 @@ import com.normation.inventory.domain.NodeId import com.normation.rudder.api.ApiVersion import com.normation.rudder.domain.logger.TimingDebugLogger import com.normation.rudder.domain.logger.TimingDebugLoggerPure +import com.normation.rudder.domain.nodes.NodeInfo import com.normation.rudder.domain.policies.Directive import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.Rule @@ -213,9 +214,10 @@ class ComplianceApi( precision <- restExtractor.extractPercentPrecision(req.params) format <- restExtractorService.extractComplianceFormat(req.params) id <- DirectiveId.parse(directiveId).toBox + d <- readDirective.getDirective(id.uid).notOptional(s"Directive with id '${id.serialize}' not found'").toBox t2 = System.currentTimeMillis _ = TimingDebugLogger.trace(s"API DirectiveCompliance - getting query param in ${t2 - t1} ms") - directive <- complianceService.getDirectiveCompliance(id, level) + directive <- complianceService.getDirectiveCompliance(d, level) t3 = System.currentTimeMillis _ = TimingDebugLogger.trace(s"API DirectiveCompliance - getting directive compliance '${id.uid.value}' in ${t3 - t2} ms") } yield { @@ -254,19 +256,15 @@ class ComplianceApi( implicit val prettify = params.prettify (for { - level <- restExtractor.extractComplianceLevel(req.params) - t1 = System.currentTimeMillis - precision <- restExtractor.extractPercentPrecision(req.params) - t2 = System.currentTimeMillis - _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting query param in ${t2 - t1} ms") - fullLibrary <- readDirective.getFullDirectiveLibrary().toBox ?~! "Could not fetch Directives" - directiveIds = fullLibrary.allDirectives.values.filter(!_._2.isSystem).map(_._2.id).toList - t3 = System.currentTimeMillis - _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting directives id ${t3 - t2} ms") - t4 = System.currentTimeMillis - directives = directiveIds.flatMap(complianceService.getDirectiveCompliance(_, level)) - t5 = System.currentTimeMillis - _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting directives compliance in ${t5 - t4} ms") + level <- restExtractor.extractComplianceLevel(req.params) + t1 = System.currentTimeMillis + precision <- restExtractor.extractPercentPrecision(req.params) + t2 = System.currentTimeMillis + _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting query param in ${t2 - t1} ms") + t4 = System.currentTimeMillis + directives <- complianceService.getDirectivesCompliance(level) + t5 = System.currentTimeMillis + _ = TimingDebugLogger.trace(s"API DirectivesCompliance - getting directives compliance in ${t5 - t4} ms") } yield { val json = directives.map( @@ -389,6 +387,124 @@ class ComplianceAPIService( val getGlobalComplianceMode: () => Box[GlobalComplianceMode] ) { + private[this] def components( + nodeInfos: Map[NodeId, NodeInfo] + )(name: String, nodeComponents: List[(NodeId, ComponentStatusReport)]): List[ByRuleComponentCompliance] = { + + val (groupsComponents, uniqueComponents) = nodeComponents.partitionMap { + case (a, b: BlockStatusReport) => Left((a, b)) + case (a, b: ValueStatusReport) => Right((a, b)) + } + + (if (groupsComponents.isEmpty) { + Nil + } else { + val bidule = groupsComponents.flatMap { case (nodeId, c) => c.subComponents.map(sub => (nodeId, sub)) } + .groupBy(_._2.componentName) + ByRuleBlockCompliance( + name, + ComplianceLevel.sum(groupsComponents.map(_._2.compliance)), + bidule.flatMap(c => components(nodeInfos)(c._1, c._2)).toList + ) :: Nil + }) ::: (if (uniqueComponents.isEmpty) { + Nil + } else { + ByRuleValueCompliance( + name, + ComplianceLevel.sum( + uniqueComponents.map(_._2.compliance) + ), // here, we finally group by nodes for each components ! + { + val byNode = uniqueComponents.groupBy(_._1) + byNode.map { + case (nodeId, components) => + ByRuleNodeCompliance( + nodeId, + nodeInfos.get(nodeId).map(_.hostname).getOrElse("Unknown node"), + ComplianceLevel.sum(components.map(_._2.compliance)), + components.sortBy(_._2.componentName).flatMap(_._2.componentValues) + ) + }.toSeq + } + ) :: Nil + }) + } + + private[this] def getByDirectivesCompliance( + directives: Seq[Directive], + level: Option[Int] + ): IOResult[Seq[ByDirectiveCompliance]] = { + val computedLevel = level.getOrElse(10) + + for { + t1 <- currentTimeMillis + + // this can be optimized, as directive only happen for level=2 + rules <- if (computedLevel >= 2) { + rulesRepo.getAll() + } else { + Seq().succeed + } + t2 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - getAllRules in ${t2 - t1} ms") + + nodeInfos <- nodeInfoService.getAll() + t3 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - nodeInfoService.getAll() in ${t3 - t2} ms") + + compliance <- getGlobalComplianceMode().toIO + t4 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - getGlobalComplianceMode in ${t4 - t3} ms") + reportsByNode <- reportingService + .findDirectiveNodeStatusReports( + nodeInfos.keySet, + directives.map(_.id).toSet + ) + .toIO + t5 <- currentTimeMillis + _ <- TimingDebugLoggerPure.trace(s"getByDirectivesCompliance - findRuleNodeStatusReports in ${t5 - t4} ms") + + } yield { + + val reportsByRule = reportsByNode.flatMap { case (_, status) => status.reports }.groupBy(_.ruleId) + val t6 = System.currentTimeMillis() + TimingDebugLoggerPure.logEffect.trace(s"getByRulesCompliance - group reports by rules in ${t6 - t5} ms") + for { + directive <- directives + } yield { + val rulesCompliance = for { + (ruleId, reportsByComponents) <- reportsByRule.toSeq.map { + case (ruleId, ruleReports) => + val reportsByComponents = (for { + ruleReport <- ruleReports + nodeId = ruleReport.nodeId + directiveReports <- ruleReport.directives.get(directive.id).toList + component <- directiveReports.components + } yield { + (nodeId, component) + }).groupBy(_._2.componentName).toSeq + (ruleId, reportsByComponents) + } + } yield { + val c = reportsByComponents.flatMap(c => components(nodeInfos)(c._1, c._2.toList)) + + val ruleName = rules.find(_.id == ruleId).map(_.name).getOrElse("") + val componentDetails = if (computedLevel <= 3) Seq() else c + + ByDirectiveByRuleCompliance(ruleId, ruleName, ComplianceLevel.sum(c.map(_.compliance)), componentDetails) + } + // level = ComplianceLevel.sum(reportsByDir.map(_.compliance)) + ByDirectiveCompliance( + directive.id, + directive.name, + ComplianceLevel.sum(rulesCompliance.map(_.compliance)), + compliance.mode, + rulesCompliance + ) + } + } + } + /** * Get the compliance for everything * level is optionnally the selected level. @@ -450,47 +566,6 @@ class ComplianceAPIService( reports.flatMap(r => r.directives.values.map(d => (r.nodeId, d)).toSeq).groupBy(_._2.directiveId) } - def components(name: String, nodeComponents: List[(NodeId, ComponentStatusReport)]): List[ByRuleComponentCompliance] = { - - val (groupsComponents, uniqueComponents) = nodeComponents.partitionMap { - case (a, b: BlockStatusReport) => Left((a, b)) - case (a, b: ValueStatusReport) => Right((a, b)) - } - - (if (groupsComponents.isEmpty) { - Nil - } else { - val bidule = groupsComponents.flatMap { case (nodeId, c) => c.subComponents.map(sub => (nodeId, sub)) } - .groupBy((_._2.componentName)) - ByRuleBlockCompliance( - name, - ComplianceLevel.sum(groupsComponents.map(_._2.compliance)), - bidule.flatMap(c => components(c._1, c._2)).toList - ) :: Nil - }) ::: (if (uniqueComponents.isEmpty) { - Nil - } else { - ByRuleValueCompliance( - name, - ComplianceLevel.sum( - uniqueComponents.map(_._2.compliance) - ), // here, we finally group by nodes for each components ! - { - val byNode = uniqueComponents.groupBy(_._1) - byNode.map { - case (nodeId, components) => - ByRuleNodeCompliance( - nodeId, - nodeInfos.get(nodeId).map(_.hostname).getOrElse("Unknown node"), - ComplianceLevel.sum(components.map(_._2.compliance)), - components.sortBy(_._2.componentName).flatMap(_._2.componentValues) - ) - }.toSeq - } - ) :: Nil - }) - } - ByRuleRuleCompliance( ruleId, ruleObjects.get(ruleId).map(_.name).getOrElse("Unknown rule"), @@ -512,7 +587,9 @@ class ComplianceAPIService( nodeDirectives.flatMap { case (nodeId, d) => d.components.map(c => (nodeId, c)).toSeq } .groupBy(_._2.componentName) } - byComponents.flatMap { case (name, nodeComponents) => components(name, nodeComponents.toList) }.toSeq + byComponents.flatMap { + case (name, nodeComponents) => components(nodeInfos)(name, nodeComponents.toList) + }.toSeq } ) }.toSeq @@ -569,34 +646,20 @@ class ComplianceAPIService( } }.toBox - def getDirectiveCompliance(directiveId: DirectiveId, level: Option[Int]): Box[ByDirectiveCompliance] = { - for { - rules <- rulesRepo.getAll() - directive <- directiveRepo.getDirective(directiveId.uid) - relevantRules = rules.filter(_.directiveIds.contains(directiveId)) - - byRules <- getByRulesCompliance(relevantRules, level) - rules = byRules.flatMap { rule => - rule.directives.map { directive => - ByDirectiveByRuleCompliance( - rule.id, - rule.name, - rule.compliance, - directive.components - ) - } - } - - compliance <- getGlobalComplianceMode().toIO + def getDirectiveCompliance(directive: Directive, level: Option[Int]): Box[ByDirectiveCompliance] = { + getByDirectivesCompliance(Seq(directive), level) + .flatMap( + _.find(_.id == directive.id).notOptional(s"No reports were found for directive with ID '${directive.id.serialize}'") + ) + .toBox + } + def getDirectivesCompliance(level: Option[Int]): Box[Seq[ByDirectiveCompliance]] = { + for { + directives <- directiveRepo.getFullDirectiveLibrary().map(_.allDirectives.values.map(_._2)) + reports <- getByDirectivesCompliance(directives.toSeq, level) } yield { - ByDirectiveCompliance( - directiveId, - directive.map(_.name).getOrElse("Unknown"), - ComplianceLevel.sum(byRules.map(_.compliance)), - compliance.mode, - rules - ) + reports } }.toBox @@ -668,10 +731,10 @@ class ComplianceAPIService( ) }.toSeq ) - }).toSeq + }) ) ) - }.toMap + } } // for each rule for each node, we want to have a @@ -701,7 +764,7 @@ class ComplianceAPIService( }) ) ) - }.toMap + } // return the full list, even for non responding nodes/directives // but override with values when available.