diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/CmdbQuery.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/CmdbQuery.scala index b8724fd7461..a0230b4d08b 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/CmdbQuery.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/CmdbQuery.scala @@ -221,7 +221,98 @@ final case object NodeStateComparator extends NodeCriterionType { ) } +final case object NodeOstypeComparator extends NodeCriterionType { + val osTypes = List("AIX", "BSD", "Linux", "Solaris", "Windows") + override def comparators = Seq(Equals, NotEquals) + override protected def validateSubCase(v:String,comparator:CriterionComparator) = { + if(null == v || v.isEmpty) Left(Inconsistency("Empty string not allowed")) else Right(v) + } + + override def matches(comparator: CriterionComparator, value: String): NodeInfoMatcher = { + comparator match { + case Equals => NodeInfoMatcher(s"Prop equals '${value}'", (node: NodeInfo) => node.osDetails.os.kernelName == value ) + case _ => NodeInfoMatcher(s"Prop not equals '${value}'", (node: NodeInfo) => node.osDetails.os.kernelName != value ) + } + } + + override def toForm(value: String, func: String => Any, attrs: (String, String)*) : Elem = + SHtml.select( + (osTypes map (e => (e,e))).toSeq + , { if(osTypes.contains(value)) Full(value) else Empty} + , func + , attrs:_* + ) +} + +final case object NodeOsNameComparator extends NodeCriterionType { + + import net.liftweb.http.S + + val osNames = AixOS :: + BsdType.allKnownTypes.sortBy { _.name } ::: + LinuxType.allKnownTypes.sortBy { _.name } ::: + (SolarisOS :: Nil) ::: + WindowsType.allKnownTypes + + + override def comparators = Seq(Equals, NotEquals) + override protected def validateSubCase(v:String,comparator:CriterionComparator) = { + if(null == v || v.isEmpty) Left(Inconsistency("Empty string not allowed")) else Right(v) + } + + override def matches(comparator: CriterionComparator, value: String): NodeInfoMatcher = { + comparator match { + case Equals => NodeInfoMatcher(s"Prop equals '${value}'", (node: NodeInfo) => node.osDetails.os.name == value ) + case _ => NodeInfoMatcher(s"Prop not equals '${value}'", (node: NodeInfo) => node.osDetails.os.name != value ) + } + } + + private[this] def distribName(x: OsType): String = { + x match { + //add linux: for linux + case _: LinuxType => "Linux - " + S.?("os.name."+x.name) + case _: BsdType => "BSD - " + S.?("os.name."+x.name) + //nothing special for windows, Aix and Solaris + case _ => S.?("os.name."+x.name) + } + } + + override def toForm(value: String, func: String => Any, attrs: (String, String)*) : Elem = + SHtml.select( + osNames.map(e => (e.name,distribName(e))).toSeq, + {osNames.find(x => x.name == value).map( _.name)}, + func, + attrs:_* + ) +} + + +final case class NodeStringComparator(access: NodeInfo => String) extends NodeCriterionType { + override val comparators = BaseComparators.comparators + + override protected def validateSubCase(v: String, comparator: CriterionComparator) = { + if(null == v || v.isEmpty) Left(Inconsistency("Empty string not allowed")) else { + comparator match { + case Regex | NotRegex => validateRegex(v) + case x => Right(v) + } + } + } + + override def matches(comparator: CriterionComparator, value: String): NodeInfoMatcher = { + + comparator match { + case Equals => NodeInfoMatcher(s"Prop equals '${value}'", (node: NodeInfo) => access(node) == value ) + case NotEquals => NodeInfoMatcher(s"Prop not equals '${value}'", (node: NodeInfo) => access(node) != value ) + case Regex => NodeInfoMatcher(s"Prop matches regex '${value}'", (node: NodeInfo) => access(node).matches(value) ) + case NotRegex => NodeInfoMatcher(s"Prop matches not regex '${value}'", (node: NodeInfo) => !access(node).matches(value) ) + case Exists => NodeInfoMatcher(s"Prop exists", (node: NodeInfo) => access(node).nonEmpty ) + case NotExists => NodeInfoMatcher(s"Prop doesn't exists", (node: NodeInfo) => access(node).isEmpty ) + } + } + +} /* * This comparator is used for "node properties"-like attribute, i.e: @@ -552,82 +643,6 @@ final case object MachineComparator extends LDAPCriterionType { ) } -final case object OstypeComparator extends LDAPCriterionType { - val osTypes = List("AIX", "BSD", "Linux", "Solaris", "Windows") - override def comparators = Seq(Equals, NotEquals) - override protected def validateSubCase(v:String,comparator:CriterionComparator) = { - if(null == v || v.isEmpty) Left(Inconsistency("Empty string not allowed")) else Right(v) - } - override def toLDAP(value:String) = Right(value) - - override def buildFilter(attributeName:String,comparator:CriterionComparator,value:String) : Filter = { - val v = value match { - case "Windows" => OC_WINDOWS_NODE - case "Linux" => OC_LINUX_NODE - case "Solaris" => OC_SOLARIS_NODE - case "AIX" => OC_AIX_NODE - case "BSD" => OC_BSD_NODE - case _ => OC_UNIX_NODE - } - comparator match { - //for equals and not equals, check value for jocker - case Equals => IS(v) - case _ => NOT(IS(v)) - } - } - - override def toForm(value: String, func: String => Any, attrs: (String, String)*) : Elem = - SHtml.select( - (osTypes map (e => (e,e))).toSeq - , { if(osTypes.contains(value)) Full(value) else Empty} - , func - , attrs:_* - ) -} - -final case object OsNameComparator extends LDAPCriterionType { - import net.liftweb.http.S - - val osNames = AixOS :: - BsdType.allKnownTypes.sortBy { _.name } ::: - LinuxType.allKnownTypes.sortBy { _.name } ::: - (SolarisOS :: Nil) ::: - WindowsType.allKnownTypes - - override def comparators = Seq(Equals, NotEquals) - override protected def validateSubCase(v:String,comparator:CriterionComparator) = { - if(null == v || v.isEmpty) Left(Inconsistency("Empty string not allowed")) else Right(v) - } - override def toLDAP(value:String) = Right(value) - - override def buildFilter(attributeName:String,comparator:CriterionComparator,value:String) : Filter = { - val osName = comparator match { - //for equals and not equals, check value for jocker - case Equals => EQ(A_OS_NAME, value) - case _ => NOT(EQ(A_OS_NAME, value)) - } - AND(EQ(A_OC,OC_NODE),osName) - } - - private[this] def distribName(x: OsType): String = { - x match { - //add linux: for linux - case _: LinuxType => "Linux - " + S.?("os.name."+x.name) - case _: BsdType => "BSD - " + S.?("os.name."+x.name) - //nothing special for windows, Aix and Solaris - case _ => S.?("os.name."+x.name) - } - } - - override def toForm(value: String, func: String => Any, attrs: (String, String)*) : Elem = - SHtml.select( - osNames.map(e => (e.name,distribName(e))).toSeq, - {osNames.find(x => x.name == value).map( _.name)}, - func, - attrs:_* - ) -} - /* * Agent comparator is kind of scpecial, because it needs to accomodate to the following cases: * - historically, agent names were only "Nova" and "Community" (understood "cfengine", of course) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/DitQueryData.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/DitQueryData.scala index 880d0c4fc17..ec4e4eee293 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/DitQueryData.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/queries/DitQueryData.scala @@ -190,10 +190,10 @@ class DitQueryData(dit: InventoryDit, nodeDit: NodeDit, rudderDit: RudderDit, ge Criterion(A_MEMORY_CAPACITY, MemoryComparator) )), ObjectCriterion(OC_NODE, Seq( - Criterion("OS",OstypeComparator) - , Criterion(A_NODE_UUID, StringComparator) - , Criterion(A_HOSTNAME, StringComparator) - , Criterion(A_OS_NAME,OsNameComparator) + Criterion("OS",NodeOstypeComparator) + , Criterion(A_NODE_UUID, NodeStringComparator(node => node.node.id.value)) + , Criterion(A_HOSTNAME, NodeStringComparator(node => node.hostname)) + , Criterion(A_OS_NAME,NodeOsNameComparator) , Criterion(A_OS_FULL_NAME, OrderedStringComparator) , Criterion(A_OS_VERSION, OrderedStringComparator) , Criterion(A_OS_SERVICE_PACK, OrderedStringComparator) @@ -205,9 +205,9 @@ class DitQueryData(dit: InventoryDit, nodeDit: NodeDit, rudderDit: RudderDit, ge , Criterion(A_AGENTS_NAME, AgentComparator) , Criterion(A_ACCOUNT, StringComparator) , Criterion(A_LIST_OF_IP, StringComparator) - , Criterion(A_ROOT_USER, StringComparator) + , Criterion(A_ROOT_USER, NodeStringComparator(node => node.localAdministratorAccountName)) , Criterion(A_INVENTORY_DATE, DateComparator) - , Criterion(A_POLICY_SERVER_UUID, StringComparator) + , Criterion(A_POLICY_SERVER_UUID, NodeStringComparator(node => node.policyServerId.value)) )), ObjectCriterion(OC_SOFTWARE, Seq( Criterion(A_NAME, StringComparator), diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala index d115a1bb913..4d5ee26df06 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala @@ -752,28 +752,37 @@ class WoLDAPNodeGroupRepository( con <- ldap existing <- getSGEntry(con, nodeGroup.id).notOptional("Error when trying to check for existence of group with id %s. Can not update".format(nodeGroup.id)) oldGroup <- mapper.entry2NodeGroup(existing).toIO.chainError("Error when trying to check for the group %s".format(nodeGroup.id.value)) - systemCheck <- if(onlyUpdateNodes) { - oldGroup.succeed - } else (oldGroup.isSystem, systemCall) match { - case (true, false) => "System group '%s' (%s) can not be modified".format(oldGroup.name, oldGroup.id.value).fail - case (false, true) => "You can not modify a non system group (%s) with that method".format(oldGroup.name).fail - case _ => oldGroup.succeed + // check if old group is the same as new group + result <- if (oldGroup.equals(nodeGroup)) { + None.succeed + } else { + for { + systemCheck <- if (onlyUpdateNodes) { + oldGroup.succeed + } else (oldGroup.isSystem, systemCall) match { + case (true, false) => "System group '%s' (%s) can not be modified".format(oldGroup.name, oldGroup.id.value).fail + case (false, true) => "You can not modify a non system group (%s) with that method".format(oldGroup.name).fail + case _ => oldGroup.succeed + } + name <- checkNameAlreadyInUse(con, nodeGroup.name, nodeGroup.id) + exists <- ZIO.when(name && !onlyUpdateNodes) { + s"Cannot change the group name to ${nodeGroup.name} : there is already a group with the same name".fail + } + onlyNodes <- if (!onlyUpdateNodes) { + UIO.unit + } else { //check that nothing but the node list changed + if (nodeGroup.copy(serverList = oldGroup.serverList) == oldGroup) { + UIO.unit + } else { + logPure.debug(s"Inconsistency when modifying node lists for nodeGroup ${nodeGroup.name}: previous content was ${oldGroup}, new is ${nodeGroup} - only the node list should change") *> + "The group configuration changed compared to the reference group you want to change the node list for. Aborting to preserve consistency".fail + } + } + change <- saveModifyNodeGroupDiff(existing, con, nodeGroup, modId, actor, reason) + } yield { + change + } } - name <- checkNameAlreadyInUse(con, nodeGroup.name, nodeGroup.id) - exists <- ZIO.when(name && !onlyUpdateNodes) { - s"Cannot change the group name to ${nodeGroup.name} : there is already a group with the same name".fail - } - onlyNodes <- if(!onlyUpdateNodes) { - UIO.unit - } else { //check that nothing but the node list changed - if(nodeGroup.copy(serverList = oldGroup.serverList) == oldGroup) { - UIO.unit - } else { - logPure.debug(s"Inconsistency when modifying node lists for nodeGroup ${nodeGroup.name}: previous content was ${oldGroup}, new is ${nodeGroup} - only the node list should change") *> - "The group configuration changed compared to the reference group you want to change the node list for. Aborting to preserve consistency".fail - } - } - result <- saveModifyNodeGroupDiff(existing, con, nodeGroup, modId, actor, reason) } yield { result } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/nodes/NodeInfoService.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/nodes/NodeInfoService.scala index a79588c7eeb..0bffee655fe 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/nodes/NodeInfoService.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/nodes/NodeInfoService.scala @@ -55,8 +55,6 @@ import com.normation.rudder.repository.CachedRepository import com.normation.inventory.ldap.core.InventoryDit import com.normation.inventory.ldap.core.InventoryMapper import com.normation.rudder.domain.Constants -import com.normation.rudder.domain.queries.CriterionComposition -import com.normation.rudder.domain.queries.NodeInfoMatcher import com.normation.ldap.sdk.LDAPIOResult._ import zio._ import zio.syntax._ @@ -65,11 +63,9 @@ import com.normation.zio._ import com.normation.ldap.sdk.syntax._ import com.normation.rudder.domain.logger.NodeLoggerPure import com.normation.rudder.domain.logger.TimingDebugLoggerPure -import com.normation.rudder.domain.queries.And import com.normation.rudder.services.nodes.NodeInfoService.A_MOD_TIMESTAMP import com.normation.rudder.services.nodes.NodeInfoServiceCached.UpdatedNodeEntries import com.normation.rudder.services.nodes.NodeInfoServiceCached.buildInfoMaps -import com.normation.rudder.services.queries.PostFilterNodeFromInfoService import scala.concurrent.duration.FiniteDuration import scala.collection.mutable.{Map => MutMap} @@ -111,12 +107,6 @@ final case class LDAPNodeInfo( trait NodeInfoService { - /** - * Retrieve minimal information needed for the node info, used (only) by the - * LDAP QueryProcessor. - */ - def getLDAPNodeInfo(nodeIds: Set[NodeId], predicates: Seq[NodeInfoMatcher], composition: CriterionComposition) : IOResult[Set[LDAPNodeInfo]] - /** * Return a NodeInfo from a NodeId. First check the ou=Node, then fetch the other data */ @@ -157,6 +147,13 @@ trait NodeInfoService { */ def getAllNodes() : IOResult[Map[NodeId, Node]] + /** + * Get all nodes + * This returns a Seq for performance reasons - it is much faster + * to return a Seq than a Set, and for subsequent use it is also + * faster + */ + def getAllNodeInfos():IOResult[Seq[NodeInfo]] /** * Get all systen node ids, for example * policy server node ids. @@ -867,15 +864,8 @@ trait NodeInfoServiceCached extends NodeInfoService with NamedZioLogger with Cac cache.view.mapValues(_._2.node).toMap.succeed } - override def getLDAPNodeInfo(nodeIds: Set[NodeId], predicates: Seq[NodeInfoMatcher], composition: CriterionComposition): IOResult[Set[LDAPNodeInfo]] = { - // if nodeIds is empty and composition is and, return an empty set; with or, we need to run it in all cases - if (nodeIds.isEmpty && composition == And) { - Set[LDAPNodeInfo]().succeed - } else { - withUpToDateCache(s"${nodeIds.size} ldap node info") { cache => - PostFilterNodeFromInfoService.getLDAPNodeInfo(nodeIds, predicates, composition, cache).succeed - } - } + def getAllNodeInfos():IOResult[Seq[NodeInfo]] = withUpToDateCache("all nodeinfos") { cache => + cache.view.values.map(_._2).toSeq.succeed } def getNodeInfo(nodeId: NodeId): IOResult[Option[NodeInfo]] = withUpToDateCache(s"${nodeId.value} node info") { cache => diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupUpdaterService.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupUpdaterService.scala index 1ee7b05f2b9..5a7cb7bcec6 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupUpdaterService.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupUpdaterService.scala @@ -98,12 +98,10 @@ class DynGroupUpdaterServiceImpl( timePreCompute = System.currentTimeMillis query <- Box(group.query) ?~! s"No query defined for group '${group.name}' (${group.id.value})" newMembers <- queryProcessor.processOnlyId(query) ?~! s"Error when processing request for updating dynamic group '${group.name}' (${group.id.value})" - //save - newMemberIdsSet = newMembers.toSet timeGroupCompute = (System.currentTimeMillis - timePreCompute) _ = logger.debug(s"Dynamic group ${group.id.value} with name ${group.name} computed in ${timeGroupCompute} ms") } yield { - group.copy(serverList = newMemberIdsSet) + group.copy(serverList = newMembers) } } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/LdapQueryProcessor.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/LdapQueryProcessor.scala index 21e35fbe6c1..cbc93fe9628 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/LdapQueryProcessor.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/LdapQueryProcessor.scala @@ -37,7 +37,6 @@ package com.normation.rudder.services.queries import java.util.regex.Pattern - import com.normation.inventory.domain._ import com.normation.inventory.ldap.core.LDAPConstants._ import com.normation.inventory.ldap.core._ @@ -47,8 +46,7 @@ import com.normation.rudder.domain._ import com.normation.rudder.domain.nodes.NodeInfo import com.normation.rudder.domain.queries._ import com.normation.rudder.repository.ldap.LDAPEntityMapper -import com.normation.rudder.services.nodes.LDAPNodeInfo -import com.normation.rudder.services.nodes.NodeInfoService +import com.normation.rudder.services.nodes.{NodeInfoService, NodeInfoServiceCached} import com.normation.utils.Control.sequence import com.unboundid.ldap.sdk.DereferencePolicy.NEVER import com.unboundid.ldap.sdk.{LDAPConnection => _, SearchScope => _, _} @@ -64,6 +62,7 @@ import com.normation.errors._ import zio._ import zio.syntax._ import com.normation.ldap.sdk.syntax._ +import com.normation.zio.currentTimeMillis /* * We have two type of filters: @@ -111,6 +110,8 @@ final case class LDAPNodeQuery( //that map MUST not contains node related filters , objectTypesFilters: Map[DnType, Map[String, List[SubQuery]]] , nodeInfoFilters : Seq[NodeInfoMatcher] + , noFilterButTakeAllFromCache: Boolean // this is when we don't have ldapfilter, but only nodeinfo filter, so we + // need to take eveything from cache ) /* @@ -121,15 +122,6 @@ final case class LDAPNodeQuery( */ final case class SubQuery(subQueryId: String, dnType: DnType, objectTypeName: String, filters: Set[ExtendedFilter]) - -final case class LdapQueryProcessorResult( - // list of entries from inventory matching the search - entries : Seq[LDAPEntry] - // a post filter to run on node info - , nodeFilters: Seq[NodeInfoMatcher] -) - - case class RequestLimits ( val subRequestTimeLimit:Int, val subRequestSizeLimit:Int, @@ -148,7 +140,7 @@ class AcceptedNodesLDAPQueryProcessor( nodeDit : NodeDit , inventoryDit : InventoryDit , processor : InternalLDAPQueryProcessor - , nodeInfoService: NodeInfoService + , nodeInfoService: NodeInfoServiceCached ) extends QueryProcessor with Loggable { private[this] case class QueryResult( @@ -168,93 +160,64 @@ class AcceptedNodesLDAPQueryProcessor( query:QueryTrait, select:Seq[String], limitToNodeIds:Option[Seq[NodeId]] - ) : Box[Seq[QueryResult]] = { + ) : Box[Set[NodeInfo]] = { val debugId = if(logger.isDebugEnabled) Helpers.nextNum else 0L val timePreCompute = System.currentTimeMillis for { - res <- processor.internalQueryProcessor(query,select,limitToNodeIds,debugId).toBox + foundNodes <- processor.internalQueryProcessor(query,select,limitToNodeIds,debugId, () => nodeInfoService.getAllNodeInfos).toBox timeres = (System.currentTimeMillis - timePreCompute) - _ = logger.debug(s"LDAP result: ${res.entries.size} entries obtained in ${timeres}ms for query ${query.toString}") - ldapEntries <- nodeInfoService.getLDAPNodeInfo(res.entries.flatMap(x => x(A_NODE_UUID).map(NodeId(_))).toSet, res.nodeFilters, query.composition).toBox - ldapEntryTime = (System.currentTimeMillis - timePreCompute - timeres) - _ = logger.debug(s"[post-filter:rudderNode] Found ${ldapEntries.size} nodes when filtering for info service existence and properties (${ldapEntryTime} ms)") - + _ = logger.debug(s"LDAP result: ${foundNodes.size} entries obtained in ${timeres}ms for query ${query.toString}") } yield { - - val inNodes = ldapEntries.map { case LDAPNodeInfo(nodeEntry, nodeInv, machineInv) => - QueryResult(nodeEntry, nodeInv, machineInv) - } - - if(logger.isDebugEnabled) { - val filtered = res.entries.map( _(A_NODE_UUID).get ).toSet -- inNodes.flatMap { case QueryResult(e, _, _) => e(A_NODE_UUID) }.toSet - if(filtered.nonEmpty) { - logger.debug(s"[${debugId}] [post-filter:rudderNode] ${inNodes.size} results (following nodes not in ou=Nodes,cn=rudder-configuration or not matching filters on NodeInfo: ${filtered.mkString(", ")}") - } - } - //filter out Rudder server component if necessary - query.returnType match { case NodeReturnType => // we have a special case for the root node that always never to that group, even if some weird // scenario lead to the removal (or non addition) of them - val withoutServerRole = inNodes.filterNot { case QueryResult(e, inv, _) => e(A_NODE_UUID) == Some("root") } + val withoutServerRole = foundNodes.filterNot { case x: NodeInfo => x.node.id.value ==("root") } if(logger.isDebugEnabled) { - val filtered = (inNodes.flatMap { case QueryResult(e, _, _) => e(A_NODE_UUID) }).toSet -- withoutServerRole.flatMap { case QueryResult(e, _, _) => e(A_NODE_UUID) } + val filtered = foundNodes.filter { case x: NodeInfo => x.node.id.value ==("root") } if(!filtered.isEmpty) { logger.debug("[%s] [post-filter:policyServer] %s results".format(debugId, withoutServerRole.size, filtered.mkString(", "))) } } - withoutServerRole.toSeq - case NodeAndPolicyServerReturnType => inNodes.toSeq + withoutServerRole + case NodeAndPolicyServerReturnType => foundNodes } } } - override def process(query:QueryTrait) : Box[Seq[NodeInfo]] = { + override def process(query:QueryTrait) : Box[Set[NodeInfo]] = { //only keep the one of the form Full(...) - queryAndChekNodeId(query, NodeInfoService.nodeInfoAttributes, None).map { seq => seq.flatMap { - case QueryResult(nodeEntry, inventoryEntry,machine) => - processor.ldapMapper.convertEntriesToNodeInfos(nodeEntry, inventoryEntry,machine).toBox match { - case Full(nodeInfo) => Seq(nodeInfo) - case e:EmptyBox => - logger.error((e ?~! "Ignoring entry in result set").messageChain) - Seq() - } - } } + queryAndChekNodeId(query, NodeInfoService.nodeInfoAttributes, None) } - override def processOnlyId(query:QueryTrait) : Box[Seq[NodeId]] = { + override def processOnlyId(query:QueryTrait) : Box[Set[NodeId]] = { //only keep the one of the form Full(...) - queryAndChekNodeId(query, Seq(A_NODE_UUID), None).map { seq => seq.flatMap { - case QueryResult(nodeEntry, _ , _) => - nodeDit.NODES.NODE.idFromDn(nodeEntry.dn) match { - case Some(nodeId) => Some(nodeId) - case None => - logger.error(s"Error when processing query ${query.toJSONString}: fetched node entry ${nodeEntry.toString()} is not a correct nodeId") - None - } - } } + queryAndChekNodeId(query, Seq(A_NODE_UUID), None).map(seq => seq.map( y => y.node.id)) } } /** - * This is the last step of query, where we are looking for high level properties check - most importantly, + * This is the last step of query, where we are looking for check in NodeInfo - and complex, * the json path and check on node properties. - * See NodeInfoService#getLDAPNodeInfo */ object PostFilterNodeFromInfoService { val logger = LoggerFactory.getLogger("com.normation.rudder.services.queries") - def getLDAPNodeInfo(nodeIds: Set[NodeId], predicates: Seq[NodeInfoMatcher], composition: CriterionComposition, nodes: Map[NodeId, (LDAPNodeInfo, NodeInfo)]): Set[LDAPNodeInfo] = { + def getLDAPNodeInfo( + foundNodeInfos: Set[NodeInfo] // the one from the search, need to be a Set for fast "contain" + , predicates : Seq[NodeInfoMatcher] + , composition : CriterionComposition + , allNodesInfos : Seq[NodeInfo] // all the nodeinfo there is + ): Set[NodeInfo] = { def comp(a: Boolean, b: Boolean) = composition match { case And => a && b case Or => a || b } - // utliity to combine predicates according to comp + // utility to combine predicates according to comp def combine(a: NodeInfoMatcher, b: NodeInfoMatcher) = new NodeInfoMatcher { override val debugString = { val c = composition match { @@ -266,16 +229,24 @@ object PostFilterNodeFromInfoService { override def matches(node: NodeInfo): Boolean = comp(a.matches(node), b.matches(node)) } + val foundNodeIds = foundNodeInfos.map(_.id) + + // if there is no predicates (ie no specific filter on NodeInfo), we should just keep nodes from our list - def predicate(nodeInfo : NodeInfo, pred: Seq[NodeInfoMatcher]) = { - val contains = nodeIds.contains(nodeInfo.id) + def predicate(nodeInfo : NodeInfo, pred: Seq[NodeInfoMatcher], composition: CriterionComposition) = { + val contains = composition match { + case Or => foundNodeIds.contains(nodeInfo.id) // we combine with all + case And => true // we combine with all + } + if (pred.isEmpty) { // in that case, whatever the query composition, we can only return what was already found. contains } else { - val combined = pred.reduceLeft(combine) - val validPredicates = combined.matches(nodeInfo) + val combined = predicates.reduceLeft(combine) + val validPredicates = combined.matches(nodeInfo) val res = comp(contains, validPredicates) + if(logger.isTraceEnabled()) { logger.trace(s"${nodeInfo.id.value}: ${if(res) "OK" else "NOK"} for [${combined.debugString}]") } @@ -283,7 +254,18 @@ object PostFilterNodeFromInfoService { } } - nodes.collect { case (_, (x, y)) if predicate(y, predicates) => x }.toSet + composition match { + // TODO: there is surely something we can do here + case Or => allNodesInfos.filter { nodeinfo => predicate(nodeinfo, predicates, composition) }.toSet + case And => + if (predicates.isEmpty) { + // paththru + foundNodeInfos + } else { + foundNodeInfos.filter { nodeinfo => predicate(nodeinfo, predicates, composition) } + } + } + } } @@ -294,23 +276,37 @@ object PostFilterNodeFromInfoService { * for pending nodes */ class PendingNodesLDAPQueryChecker( - val checker:InternalLDAPQueryProcessor -) extends QueryChecker { + val checker:InternalLDAPQueryProcessor + , nodeInfoService: NodeInfoService +) extends QueryChecker with Loggable { - override def check(query:QueryTrait, limitToNodeIds:Option[Seq[NodeId]]) : Box[Seq[NodeId]] = { + override def check(query:QueryTrait, limitToNodeIds:Option[Seq[NodeId]]) : Box[Set[NodeId]] = { if(query.criteria.isEmpty) { LoggerFactory.getILoggerFactory.getLogger(Logger.loggerNameFor(classOf[InternalLDAPQueryProcessor])).debug( s"Checking a query with 0 criterium will always lead to 0 nodes: ${query}" ) - Full(Seq.empty[NodeId]) + Full(Set.empty[NodeId]) } else { + val timePreCompute = System.currentTimeMillis for { - res <- checker.internalQueryProcessor(query, Seq("1.1"), limitToNodeIds).toBox - ids <- sequence(res.entries) { entry => - checker.ldapMapper.nodeDn2OptNodeId(entry.dn).toBox ?~! "Can not get node ID from dn %s".format(entry.dn) - } + // get the pending node infos we are considering + allPendingNodeInfos <- nodeInfoService.getPendingNodeInfos().toBox + pendingNodeInfos = limitToNodeIds match { + case None => allPendingNodeInfos.values.toSeq + case Some(ids) => allPendingNodeInfos.collect { case (nodeId, nodeInfo) if ids.contains(nodeId) => nodeInfo}.toSeq + } + foundNodes <- checker.internalQueryProcessor(query, Seq("1.1"), limitToNodeIds, 0, () => pendingNodeInfos.succeed).toBox + timeres = (System.currentTimeMillis - timePreCompute) + _ = logger.debug(s"LDAP result: ${foundNodes.size} entries in pending nodes obtained in ${timeres}ms for query ${query.toString}") } yield { - ids + //filter out Rudder server component if necessary + (query.returnType match { + case NodeReturnType => + // we have a special case for the root node that always never to that group, even if some weird + // scenario lead to the removal (or non addition) of them + foundNodes.filterNot { case x: NodeInfo => x.node.id.value ==("root") } + case NodeAndPolicyServerReturnType => foundNodes + }).map(_.node.id) } } } @@ -347,17 +343,18 @@ class InternalLDAPQueryProcessor( /** * - * The high level query processor, with all the - * relevant logics. - * Sub classes should call that method to - * implement process&check method + * The high level query processor, with all the relevant logics. + * It looks in LDAP for infos that are only there + * and in the NodeInfos for eveything else */ def internalQueryProcessor( query : QueryTrait , select : Seq[String] = Seq() , limitToNodeIds: Option[Seq[NodeId]] = None , debugId : Long = 0L - ) : IOResult[LdapQueryProcessorResult] = { + , lambdaAllNodeInfos : () => IOResult[Seq[NodeInfo]] // this is hackish, to have the list of all node if + // only if necessary, to avoid the overall cost of looking for it + ) : IOResult[Set[NodeInfo]] = { @@ -467,20 +464,54 @@ class InternalLDAPQueryProcessor( for { //log start query - _ <- logPure.debug(s"[${debugId}] Start search for ${query.toString}") + _ <- logPure.debug(s"[${debugId}] Start search for ${query.toString}") + timeStart <- currentTimeMillis // Construct & normalize the data nq <- normalizedQuery - lots <- ldapObjectTypeSets(nq) - dmms <- dnMapMapSets(nq, lots) - optdms = dnMapSets(nq, dmms) + // special case: no query, but we create a dummy one, + // identified by noFilterButTakeAllFromCache = true + // in this case, we skip all the ldap part + optdms <- { + if (nq.noFilterButTakeAllFromCache) { + None.succeed + } else { + + for { + lots <- ldapObjectTypeSets(nq) + dmms <- dnMapMapSets(nq, lots) + inneroptdms = dnMapSets(nq, dmms) + } yield { + inneroptdms + } + } + } + + // Fetching all node infos if necessary + // This is an optimisation procedue, as it is a bit costly to fetch it, so we want to + // have it only if the query is an OR, and Invertion, or and AND and there ae + // no LDAP criteria + allNodesInfos <- query.composition match { + case Or => lambdaAllNodeInfos() + case And if nq.noFilterButTakeAllFromCache => lambdaAllNodeInfos() + case And if query.transform == ResultTransformation.Invert => lambdaAllNodeInfos() + case And if optdms.isDefined => lambdaAllNodeInfos() + + case _ => Seq[NodeInfo]().succeed + } + timefetch <- currentTimeMillis + _ <- logPure.debug(s"LDAP result: fetching if necessary all nodesInfos in in nodes obtained in ${timefetch-timeStart} ms for query ${query.toString}") + // If dnMapSets returns a None, then it means that we are ANDing composition with an empty value // so we skip the last query results <- optdms match { - case None => - Seq[LDAPEntry]().succeed + case None if nq.noFilterButTakeAllFromCache => + allNodesInfos.toSet.succeed + case None => + Set[NodeInfo]().succeed case Some(dms) => (for { // Ok, do the computation here + // still rely on LDAP here _ <- logPure.ifTraceEnabled { ZIO.foreach_(dms) { case (dnType, dns) => logPure.trace(s"/// ${dnType} ==> ${dns.map(_.getRDN).mkString(", ")}") @@ -496,41 +527,65 @@ class InternalLDAPQueryProcessor( rt = nodeObjectTypes.copy(filter = finalLdapFilter) _ <- logPure.debug(s"[${debugId}] |- (final query) ${rt}") entries <- executeQuery(rt.baseDn, rt.scope, nodeObjectTypes.objectFilter, rt.filter, finalSpecialFilters, select.toSet, nq.composition, debugId) - } yield entries). + // convert these entries into nodeInfo + nodesId = entries.flatMap(x => x(A_NODE_UUID)).toSet + nodeInfos = allNodesInfos.filter(nodeInfo => nodesId.contains(nodeInfo.node.id.value)).toSet + } yield nodeInfos). tapError(err => logPure.debug(s"[${debugId}] `-> error: ${err.fullMsg}")). tap(seq => logPure.debug(s"[${debugId}] `-> ${seq.size} results")) } - inverted <- query.transform match { - case ResultTransformation.Identity => results.succeed + // No more LDAP query is required here + // Do the filtering about non LDAP data here + timeldap <- currentTimeMillis + _ <- logPure.debug(s"LDAP result: ${results.size} entries in nodes obtained in ${timeldap-timeStart} ms for query ${query.toString}") + + + nodeInfoFiltered = query.composition match { + case And if results.isEmpty => + // And and nothing returns nothing + Set[NodeInfo]() + case And => + // If i'm doing and AND, there is no need for the allNodes here + PostFilterNodeFromInfoService.getLDAPNodeInfo(results, nq.nodeInfoFilters, query.composition, Seq()) + case Or => + // Here we need the list of all nodes + PostFilterNodeFromInfoService.getLDAPNodeInfo(results, nq.nodeInfoFilters, query.composition, allNodesInfos) + } + timefilter <- currentTimeMillis + _ <- logPure.debug(s"[post-filter:rudderNode] Found ${nodeInfoFiltered.size} nodes when filtering for info service existence and properties (${timefilter-timeldap} ms)") + _ <- logPure.ifDebugEnabled{ + val filtered = results.map( x => x.node.id.value).diff(nodeInfoFiltered.map( x => x.node.id.value)) + if(filtered.nonEmpty) { + logPure.debug(s"[${debugId}] [post-filter:rudderNode] ${nodeInfoFiltered.size} results (following nodes not in ou=Nodes,cn=rudder-configuration or not matching filters on NodeInfo: ${filtered.mkString(", ")}") + } else { + logPure.debug(s"[${debugId}] [post-filter:rudderNode] ${nodeInfoFiltered.size} results (following nodes not in ou=Nodes,cn=rudder-configuration or not matching filters on NodeInfo: ${filtered.mkString(", ")}") + + } + } + + inverted = query.transform match { + case ResultTransformation.Identity => nodeInfoFiltered case ResultTransformation.Invert => - for { - _ <- logPure.debug(s"[${debugId}] |- (need to get all nodeIds for inversion) ") - allIds <- executeQuery(nodeObjectTypes.baseDn, nodeObjectTypes.scope, nodeObjectTypes.objectFilter, Some(ALL), Set(), Set(), nq.composition, debugId) - ids = results.flatMap(x => x(A_NODE_UUID)).toSet - res = allIds.filterNot( e => ids.contains(e.value_!(A_NODE_UUID)) ) - _ <- logPure.debug(s"[${debugId}] |- (invert) entries after inversion: ${res.size}") - } yield { + logEffect.debug(s"[${debugId}] |- (need to get all nodeIds for inversion) ") + val res = allNodesInfos.toSet.diff(nodeInfoFiltered) + logEffect.debug(s"[${debugId}] |- (invert) entries after inversion: ${res.size}") res } - } - // distinctBy computes the hashcode of the object - // It is really expensive on LDAP entries. - // The dn string is already computed, so the toString is a good alternative (and as also unicity) - postFiltered = postFilterNode(inverted.distinctBy(_.dn.toString), query.returnType, limitToNodeIds) + postFiltered = postFilterNode(inverted, query.returnType, limitToNodeIds) } yield { - LdapQueryProcessorResult(postFiltered, nq.nodeInfoFilters) + postFiltered } } /** - * That method allows to post-process a list of nodes based on + * That method allows to post-process a list of NodeInfo based on * the resultType. * - step1: ~filter out policy server if we only want "simple" nodes~ => no, we need to do * that in `queryAndChekNodeId` where we know about server roles * - step2: filter out nodes based on a given list of acceptable entries */ - private[this] def postFilterNode(entries: Seq[LDAPEntry], returnType: QueryReturnType, limitToNodeIds:Option[Seq[NodeId]]) : Seq[LDAPEntry] = { - + private[this] def postFilterNode(entries: Set[NodeInfo], returnType: QueryReturnType, limitToNodeIds:Option[Seq[NodeId]]) : Set[NodeInfo] = { +// keeping this for backport option to 6.2 val step1 = returnType match { //actually, we are able at that point to know if we have a policy server, //so we don't post-process anything. @@ -539,8 +594,8 @@ class InternalLDAPQueryProcessor( } val step2 = limitToNodeIds match { case None => step1 - case Some(seq) => step1.filter(e => - seq.exists(nodeId => nodeId.value == e(A_NODE_UUID).getOrElse("Missing attribute %s in node entry, that case is not supported.").format(A_NODE_UUID)) + case Some(seq) => step1.filter(nodeInfo => + seq.exists(nodeId => nodeId.value == nodeInfo.node.id.value) ) } @@ -973,17 +1028,20 @@ class InternalLDAPQueryProcessor( subQueries = groupedSetFilter.view.mapValues(_.view.filterKeys { _ != "node" }.toMap).filterNot( _._2.isEmpty).toMap } yield { // at that point, it may happen that nodeFilters and otherFilters are empty - val mainFilters = if(groupedSetFilter.isEmpty) { + val (mainFilters, andAndEmpty) = if(groupedSetFilter.isEmpty) { query.composition match { // In that case, we add a "get all nodes" query and all filters will be done in node info. - case And => Some(Set[ExtendedFilter](LDAPFilter(BuildFilter.ALL))) + // we should have a specific case here, saying: it's empty, we are AND, so we AND with all the cache +// case And => (None, true) + + case And => (Some(Set[ExtendedFilter](LDAPFilter(BuildFilter.ALL))), true) // In that case, We should return no Nodes from this query and we will only query nodes with filters from node info. - case Or => None + case Or => (None, false) } - } else { nodeFilters } + } else { (nodeFilters, false) } val nodeInfos = nodeInfoFilters.map { case QueryFilter.NodeInfo(c, comp, value) => c.matches(comp, value)} - LDAPNodeQuery(mainFilters, query.composition, query.transform, subQueries, nodeInfos) + LDAPNodeQuery(mainFilters, query.composition, query.transform, subQueries, nodeInfos, andAndEmpty) } } } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/QueryProcessor.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/QueryProcessor.scala index 72c45b864c0..fb14f2371e6 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/QueryProcessor.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/queries/QueryProcessor.scala @@ -50,13 +50,13 @@ trait QueryProcessor { * @param select - attributes to fetch in the ldap entry. If empty, all attributes are fetched * @return */ - def process(query:QueryTrait) : Box[Seq[NodeInfo]] + def process(query:QueryTrait) : Box[Set[NodeInfo]] /** * Only get node ids corresponding to that request, with minimal consistency check. * This method is useful to maximize performance (low memory, high throughout) for ex for dynamic groups. */ - def processOnlyId(query:QueryTrait) : Box[Seq[NodeId]] + def processOnlyId(query:QueryTrait) : Box[Set[NodeId]] } @@ -74,6 +74,6 @@ trait QueryChecker { * Full(seq) with seq being the list of nodeId which verify * query. */ - def check(query:QueryTrait, nodeIds:Option[Seq[NodeId]]) : Box[Seq[NodeId]] + def check(query:QueryTrait, nodeIds:Option[Seq[NodeId]]) : Box[Set[NodeId]] } diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala index 1c6f72d0fba..097e82b185f 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/jdbc/ReportingServiceTest.scala @@ -61,7 +61,6 @@ import com.normation.rudder.reports.ResolvedAgentRunInterval import com.normation.rudder.reports.GlobalComplianceMode import com.normation.rudder.reports.execution._ import com.normation.rudder.repository.{CategoryWithActiveTechniques, ComplianceRepository, FullActiveTechniqueCategory, RoDirectiveRepository, RoRuleRepository} -import com.normation.rudder.services.nodes.LDAPNodeInfo import com.normation.rudder.services.nodes.NodeInfoService import com.normation.rudder.services.policies.NodeConfigData import com.normation.rudder.services.reports.{CachedFindRuleNodeStatusReports, CachedNodeChangesServiceImpl, DefaultFindRuleNodeStatusReports, NodeChangesServiceImpl, NodeConfigurationService, NodeConfigurationServiceImpl, ReportingServiceImpl, UnexpectedReportInterpretation} @@ -95,11 +94,11 @@ class ReportingServiceTest extends DBCommon with BoxSpecMatcher { } object nodeInfoService extends NodeInfoService { - def getLDAPNodeInfo(nodeIds: Set[NodeId], predicates: Seq[NodeInfoMatcher], composition: CriterionComposition) : IOResult[Set[LDAPNodeInfo]] = ??? def getNodeInfo(nodeId: NodeId) : IOResult[Option[NodeInfo]] = ??? def getNodeInfos(nodesId: Set[NodeId]) : IOResult[Set[NodeInfo]] = ??? def getNode(nodeId: NodeId): Box[Node] = ??? def getAllNodes() : IOResult[Map[NodeId, Node]] = ??? + def getAllNodeInfos():IOResult[Seq[NodeInfo]] = ??? def getAllNodesIds(): IOResult[Set[NodeId]] = ??? def getAllSystemNodeIds() : IOResult[Seq[NodeId]] = ??? def getPendingNodeInfos(): IOResult[Map[NodeId, NodeInfo]] = ??? diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestPendingNodePolicies.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestPendingNodePolicies.scala index bdd58e19c12..cdcf4260d4c 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestPendingNodePolicies.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestPendingNodePolicies.scala @@ -158,7 +158,7 @@ class TestPendingNodePolicies extends Specification { // a fake query checker val queryChecker = new QueryChecker { - override def check(query: QueryTrait, nodeIds: Option[Seq[NodeId]]): Box[Seq[NodeId]] = { + override def check(query: QueryTrait, nodeIds: Option[Seq[NodeId]]): Box[Set[NodeId]] = { // make a 0 criteria request raise an error like LDAP would do, // see: https://www.rudder-project.org/redmine/issues/12338 if(query.criteria.isEmpty) { @@ -168,7 +168,7 @@ class TestPendingNodePolicies extends Specification { case x if(x == dummyQuery0) => Set.empty[NodeId] case x if(x == dummyQuery1) => Set(node) case x => Set(node) - }).intersect(nodeIds.getOrElse(Seq(node)).toSet).toSeq) + }).intersect(nodeIds.getOrElse(Seq(node)).toSet)) } } } diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestQueryProcessor.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestQueryProcessor.scala index acce4f65d59..35f326ecabb 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestQueryProcessor.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/queries/TestQueryProcessor.scala @@ -188,7 +188,7 @@ class TestQueryProcessor extends Loggable { """).openOrThrowException("For tests"), Nil) - testQueries( q0 :: q1 :: q2 :: Nil, true) + testQueries( q0 :: q1 :: q2 :: Nil, false) } @Test def basicQueriesOnOneNodeParameter(): Unit = { @@ -311,7 +311,61 @@ class TestQueryProcessor extends Loggable { """).openOrThrowException("For tests"), s(0) :: s(1) :: s(2) :: s(3) :: s(4) :: s(5) :: s(6) :: s(7) :: Nil) - testQueries(q1 :: q2 :: q3 :: q4 :: q5 :: q6 :: q7 :: Nil, true) + testQueries(q1 :: q2 :: q3 :: q4 :: q5 :: q6 :: q7 :: Nil, false) + } + + // group of group, with or/and composition + @Test def groupOfgroupsDoIntenalQueryTest(): Unit = { + val q1 = TestQuery( + "q1", + parser(""" + { "select":"node", "where":[ + { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node1" } + ] } + """).openOrThrowException("For tests"), + s(1) :: Nil) + + val q2 = TestQuery( + "q2", + parser(""" + { "select":"node", "composition":"or", "where":[ + { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node1" } + , { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node2" } + ] } + """).openOrThrowException("For tests"), + s(1) :: s(2) :: Nil) + + val q3 = TestQuery( + "q3", + parser(""" + { "select":"node", "where":[ + { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node1" } + , { "objectType":"node", "attribute":"ram", "comparator":"gt", "value":"1" } + ] } + """).openOrThrowException("For tests"), + s(1) :: Nil) + + val q4 = TestQuery( + "q4", + parser(""" + { "select":"node", "where":[ + { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node1" } + , { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node12" } + ] } + """).openOrThrowException("For tests"), + s(1) :: Nil) + + val q5 = TestQuery( + "q5", + parser(""" + { "select":"node", "where":[ + { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node12" } + , { "objectType":"group", "attribute":"nodeGroupId", "comparator":"eq", "value":"test-group-node23" } + ] } + """).openOrThrowException("For tests"), + s(2) :: Nil) + + testQueries(q1 :: q2 :: q3 :: q4 :: q5 :: Nil, true) } @Test def machineComponentQueries(): Unit = { @@ -388,7 +442,7 @@ class TestQueryProcessor extends Loggable { """).openOrThrowException("For tests"), s(2) :: Nil) - testQueries(q1 :: q2 :: q3 :: Nil, true) + testQueries(q1 :: q2 :: q3 :: q3bis:: Nil, true) } @Test def networkInterfaceElementQueries(): Unit = { @@ -556,6 +610,101 @@ class TestQueryProcessor extends Loggable { //s2,s3 not ok because in the "not regex" pattern //s4 ok because only 127.0.0.1 + // test query that matches a software version + val q10 = TestQuery( + "q10", + parser(""" + { "select":"node", "where":[ + { "objectType":"software", "attribute":"softwareVersion", "comparator":"regex", "value":"1\\.0.*" } + ] } + """).openOrThrowException("For tests"), + Seq(s(2), s(7)) ) + + // test "notRegex" query: "I want node for which ram is not "100000000" (ie not node1) + val q11 = TestQuery( + "q11", + parser(""" + { "select":"node", "where":[ + { "objectType":"node", "attribute":"ram", "comparator":"notRegex", "value":"100000000" } + ] } + """).openOrThrowException("For tests"), + s.filterNot(n => n == s(1)) ) + + testQueries(q0 :: q1 :: q1_ :: q2 :: q2_ :: q3 :: q3_2 :: q4 :: q5 :: q6 :: q7 :: q8 :: q9 :: q10 :: q11 :: Nil, false) + } + + @Test def regexQueriesInventories(): Unit = { + // this test if for the queries that can be performed using only LDAP + //regex and "subqueries" for logical elements should not be contradictory + //here, we have to *only* search for logical elements with the regex + //and cn is both on node and logical elements + val q0 = TestQuery( + "q0", + parser(""" + { "select":"node", "composition":"or" , "where":[ + , { "objectType":"fileSystemLogicalElement", "attribute":"description", "comparator":"regex", "value":"matchOnM[e]" } + ] } + """).openOrThrowException("For tests"), + s(3) :: Nil) + + + //on software, machine, machine element, node element + val q2 = TestQuery( + "q2", + parser(""" + { "select":"nodeAndPolicyServer", "where":[ + { "objectType":"software", "attribute":"cn", "comparator":"regex" , "value":"Software [0-9]" } + , { "objectType":"machine", "attribute":"machineId", "comparator":"regex" , "value":"machine[0-2]" } + , { "objectType":"fileSystemLogicalElement", "attribute":"fileSystemFreeSpace", "comparator":"regex", "value":"[01]{2}" } + , { "objectType":"biosPhysicalElement", "attribute":"softwareVersion", "comparator":"regex", "value":"[6.0]+" } + ] } + """).openOrThrowException("For tests"), + s(7) :: Nil) + + val q2_ = TestQuery("q2_", query = q2.query match { case q : Query => q.copy(composition = Or); case q : NewQuery => q.copy(composition = Or)}, + ( + s(2) :: s(7) :: //software + s(4) :: s(5) :: s(6) :: s(7) :: //machine + s(2) :: root :: // free space + s(2) :: //bios + Nil).distinct) + + val q5 = TestQuery( + "q5", + parser(""" + { "select":"nodeAndPolicyServer","composition":"or", "where":[ + , { "objectType":"fileSystemLogicalElement" , "attribute":"mountPoint" , "comparator":"regex", "value":"[/]" } + ] } + """).openOrThrowException("For tests"), + s(3) :: s(7) :: root :: Nil) + + //same as q5 on IP, to test with escaping + //192.168.56.101 is for node3 + val q8 = TestQuery( + "q8", + parser(""" + { "select":"node", "where":[ + { "objectType":"node" , "attribute":"ipHostNumber" , "comparator":"notRegex", "value":"192.168.56.101" } + ] } + """).openOrThrowException("For tests"), + s.filterNot( _ == s(1)) ) + + //typical use case for server on internal/dmz/both: want intenal (but not both) + //that test a match regex and not regex + val q9 = TestQuery( + "q9", + parser(""" + { "select":"node", "where":[ + { "objectType":"node" , "attribute":"ipHostNumber" , "comparator":"regex", "value":"127.0.0.*" } + , { "objectType":"node" , "attribute":"ipHostNumber" , "comparator":"notRegex", "value":"192.168.56.10[23]" } + ] } + """).openOrThrowException("For tests"), + Seq(s(1), s(4)) ) + //s0,5,6,7,8 not ok because no 127.0.0.1 + //s1 ok because not in "not regex" pattern + //s2,s3 not ok because in the "not regex" pattern + //s4 ok because only 127.0.0.1 + // test query that matches a software version val q10 = TestQuery( "q10", @@ -579,16 +728,16 @@ class TestQueryProcessor extends Loggable { // test query that doesn't match a software name, ie we want all nodes on which "software 1" is not // installed (we don't care if there is 0 or 1000 other software) // THIS DOES NOT WORK DUE TO: https://issues.rudder.io/issues/19137 -// val q12 = TestQuery( -// "q12", -// parser(""" -// { "select":"node", "composition":"or", "where":[ -// { "objectType":"software", "attribute":"cn", "comparator":"notRegex", "value":"Software 1" } -// ] } -// """).openOrThrowException("For tests"), -// s.filterNot(n => n == s(2)) ) - - testQueries(q0 :: q1 :: q1_ :: q2 :: q2_ :: q3 :: q3_2 :: q4 :: q5 :: q6 :: q7 :: q8 :: q9 :: q10 :: q11 :: Nil, true) + // val q12 = TestQuery( + // "q12", + // parser(""" + // { "select":"node", "composition":"or", "where":[ + // { "objectType":"software", "attribute":"cn", "comparator":"notRegex", "value":"Software 1" } + // ] } + // """).openOrThrowException("For tests"), + // s.filterNot(n => n == s(2)) ) + + testQueries(q0 :: q2 :: q2_ :: q5 :: q8 :: q9 :: q10 :: q11 :: Nil, true) } @Test def invertQueries(): Unit = { @@ -704,7 +853,7 @@ class TestQueryProcessor extends Loggable { """).openOrThrowException("For tests"), sr) - testQueries( q0 :: q1 :: Nil, true) + testQueries( q0 :: q1 :: Nil, false) } @Test def agentTypeQueries: Unit = { @@ -1056,7 +1205,7 @@ class TestQueryProcessor extends Loggable { private def testQueryResultProcessor(name:String,query:QueryTrait, nodes:Seq[NodeId], doInternalQueryTest : Boolean) = { val ids = nodes.sortBy( _.value ) - val found = queryProcessor.process(query).openOrThrowException("For tests").map { _.id }.sortBy( _.value ) + val found = queryProcessor.process(query).openOrThrowException("For tests").map { _.id }.toSeq.sortBy( _.value ) //also test with requiring only the expected node to check consistancy //(that should not change anything) @@ -1078,10 +1227,9 @@ class TestQueryProcessor extends Loggable { if (doInternalQueryTest) { logger.debug("Testing with expected entries, This test should be ignored when we are looking for Nodes with NodeInfo and inventory (ie when we are looking for property and environement variable") val foundWithLimit = - (internalLDAPQueryProcessor.internalQueryProcessor(query, limitToNodeIds = Some(ids)).runNow.entries.map { - entry => - NodeId(entry("nodeId").get) - }).distinct.sortBy( _.value ) + (internalLDAPQueryProcessor.internalQueryProcessor(query, limitToNodeIds = Some(ids), lambdaAllNodeInfos = (() => nodeInfoService.getAllNodeInfos())).runNow.map { + _.node.id + }).toSeq.distinct.sortBy( _.value ) assertEquals( s"[${name}] Size differs between expected and found entries (InternalQueryProcessor, only inventory fields)\n Found: ${foundWithLimit}\n Expected: ${ids}" diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala index 700dc4c2584..a8446bf0b83 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/services/reports/CachedFindRuleNodeStatusReportsTest.scala @@ -42,8 +42,6 @@ import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.nodes.Node import com.normation.rudder.domain.nodes.NodeInfo import com.normation.rudder.domain.policies.RuleId -import com.normation.rudder.domain.queries.CriterionComposition -import com.normation.rudder.domain.queries.NodeInfoMatcher import com.normation.rudder.domain.reports.ComplianceLevel import com.normation.rudder.domain.reports.NodeConfigId import com.normation.rudder.domain.reports.NodeExpectedReports @@ -53,7 +51,6 @@ import com.normation.rudder.reports.GlobalComplianceMode import com.normation.rudder.reports.execution.RoReportsExecutionRepository import com.normation.rudder.repository.FindExpectedReportRepository import com.normation.rudder.repository.ReportsRepository -import com.normation.rudder.services.nodes.LDAPNodeInfo import com.normation.rudder.services.nodes.NodeInfoService import com.normation.rudder.services.policies.NodeConfigData import net.liftweb.common.Box @@ -132,12 +129,12 @@ class CachedFindRuleNodeStatusReportsTest extends Specification { ) object testNodeInfoService extends NodeInfoService { - def getLDAPNodeInfo(nodeIds: Set[NodeId], predicates: Seq[NodeInfoMatcher], composition: CriterionComposition) : IOResult[Set[LDAPNodeInfo]] = ??? def getNodeInfo(nodeId: NodeId) : IOResult[Option[NodeInfo]] = ??? def getNodeInfos(nodesId: Set[NodeId]) : IOResult[Set[NodeInfo]] = ??? def getNode(nodeId: NodeId): Box[Node] = ??? def getAllNodes() : IOResult[Map[NodeId, Node]] = ??? def getAllNodesIds(): IOResult[Set[NodeId]] = ??? + def getAllNodeInfos():IOResult[Seq[NodeInfo]] = ??? def getAllSystemNodeIds() : IOResult[Seq[NodeId]] = ??? def getPendingNodeInfos(): IOResult[Map[NodeId, NodeInfo]] = ??? def getPendingNodeInfo(nodeId: NodeId): IOResult[Option[NodeInfo]] = ??? diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/NodeApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/NodeApi.scala index 5b031c30b22..dc70e142d21 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/NodeApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/NodeApi.scala @@ -1054,7 +1054,7 @@ class NodeApiService6 ( case _ => Failure(s"Invalid branch used for nodes query, expected either AcceptedInventory or PendingInventory, got ${state}") } } yield { - listNodes(state,detailLevel,Some(nodeIds),version) + listNodes(state,detailLevel,Some(nodeIds.toSeq),version) } ) match { case Full(resp) => { diff --git a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala index e62ad5cfbe5..d31e9d3e268 100644 --- a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala +++ b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala @@ -76,10 +76,8 @@ import com.normation.inventory.domain.AgentType.CfeCommunity import com.normation.zio._ import com.normation.rudder.domain.archives.RuleArchiveId import com.normation.rudder.domain.queries.CriterionComposition -import com.normation.rudder.domain.queries.NodeInfoMatcher import com.normation.rudder.repository.RoRuleRepository import com.normation.rudder.repository.WoRuleRepository -import com.normation.rudder.services.nodes.LDAPNodeInfo import com.normation.rudder.services.nodes.NodeInfoService import com.normation.rudder.services.policies.NodeConfiguration import com.normation.rudder.services.policies.ParameterForConfiguration @@ -1489,6 +1487,7 @@ z5VEb9yx2KikbWyChM1Akp82AV5BzqE80QIBIw== } override def getAllNodes(): IOResult[Map[NodeId, Node]] = getAll().map(_.map(kv => (kv._1, kv._2.node))) + override def getAllNodeInfos():IOResult[Seq[NodeInfo]] = getAll().map(_.values.toSeq) override def getAllNodesIds(): IOResult[Set[NodeId]] = getAllNodes().map(_.keySet) override def getAllSystemNodeIds(): IOResult[Seq[NodeId]] = { nodeBase.get.map(_.collect { case (id, n) if(n.info.isSystem) => id }.toSeq ) @@ -1513,7 +1512,6 @@ z5VEb9yx2KikbWyChM1Akp82AV5BzqE80QIBIw== override def getAllNodeInventories(inventoryStatus: InventoryStatus): IOResult[Map[NodeId, NodeInventory]] = getGenericAll(inventoryStatus, _fullInventory(_).map(_.node)) // not implemented yet - override def getLDAPNodeInfo(nodeIds: Set[NodeId], predicates: Seq[NodeInfoMatcher], composition: CriterionComposition): IOResult[Set[LDAPNodeInfo]] = ??? override def getNumberOfManagedNodes: Int = ??? override def save(serverAndMachine: FullInventory): IOResult[Seq[LDIFChangeRecord]] = ??? override def delete(id: NodeId, inventoryStatus: InventoryStatus): IOResult[Seq[LDIFChangeRecord]] = ??? @@ -1705,16 +1703,16 @@ z5VEb9yx2KikbWyChM1Akp82AV5BzqE80QIBIw== } } - override def process(query: QueryTrait): Box[Seq[NodeInfo]] = { + override def process(query: QueryTrait): Box[Set[NodeInfo]] = { for { nodes <- nodeInfoService.nodeBase.get matching <- filterForLines(query.criteria, query.composition, nodes.map(_._2).toList).toIO } yield { - matching.map(_.info) + matching.map(_.info).toSet } }.toBox - override def processOnlyId(query: QueryTrait): Box[Seq[NodeId]] = process(query).map(_.map(_.id)) + override def processOnlyId(query: QueryTrait): Box[Set[NodeId]] = process(query).map(_.map(_.id)) } } diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala index f85bd1154b9..2af28d33257 100644 --- a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala +++ b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala @@ -1666,7 +1666,8 @@ object RudderConfig extends Loggable { // here, we don't want to look for subgroups to show them in the form => always return an empty list , new DitQueryData(pendingNodesDitImpl, nodeDit, rudderDit, () => Nil.succeed) , ldapEntityMapper - ) + ), + nodeInfoServiceImpl ) private[this] lazy val dynGroupServiceImpl = new DynGroupServiceImpl(rudderDitImpl, roLdap, ldapEntityMapper) diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/com/normation/rudder/web/components/SearchNodeComponent.scala b/webapp/sources/rudder/rudder-web/src/main/scala/com/normation/rudder/web/components/SearchNodeComponent.scala index d17b98f23ea..6130c3194b3 100644 --- a/webapp/sources/rudder/rudder-web/src/main/scala/com/normation/rudder/web/components/SearchNodeComponent.scala +++ b/webapp/sources/rudder/rudder-web/src/main/scala/com/normation/rudder/web/components/SearchNodeComponent.scala @@ -192,7 +192,7 @@ class SearchNodeComponent( query = Some(newQuery) if(errors.isEmpty) { // ********* EXECUTE QUERY *********** - srvList = queryProcessor.process(newQuery) + srvList = queryProcessor.process(newQuery).map(_.toSeq) initUpdate = true searchFormHasError = false } else { @@ -610,7 +610,7 @@ object SearchNodeComponent { val defaultLine : CriterionLine = { //in case of further modification in ditQueryData require(ditQueryData.criteriaMap(OC_NODE).criteria(0).name == "OS", "Error in search node criterion default line, did you change DitQueryData ?") - require(ditQueryData.criteriaMap(OC_NODE).criteria(0).cType.isInstanceOf[OstypeComparator.type], "Error in search node criterion default line, did you change DitQueryData ?") + require(ditQueryData.criteriaMap(OC_NODE).criteria(0).cType.isInstanceOf[NodeOstypeComparator.type], "Error in search node criterion default line, did you change DitQueryData ?") CriterionLine( objectType = ditQueryData.criteriaMap(OC_NODE) , attribute = ditQueryData.criteriaMap(OC_NODE).criteria(0) diff --git a/webapp/sources/rudder/rudder-web/src/test/scala/bootstrap/liftweb/checks/migration/TestMigrateSystemTechnique7_0.scala b/webapp/sources/rudder/rudder-web/src/test/scala/bootstrap/liftweb/checks/migration/TestMigrateSystemTechnique7_0.scala index c03bc00130f..3fb6df04122 100644 --- a/webapp/sources/rudder/rudder-web/src/test/scala/bootstrap/liftweb/checks/migration/TestMigrateSystemTechnique7_0.scala +++ b/webapp/sources/rudder/rudder-web/src/test/scala/bootstrap/liftweb/checks/migration/TestMigrateSystemTechnique7_0.scala @@ -74,8 +74,6 @@ import com.normation.rudder.domain.policies.DirectiveUid import com.normation.rudder.domain.policies.ModifyRuleDiff import com.normation.rudder.domain.policies.Rule import com.normation.rudder.domain.policies.RuleId -import com.normation.rudder.domain.queries.CriterionComposition -import com.normation.rudder.domain.queries.NodeInfoMatcher import com.normation.rudder.domain.workflows.ChangeRequestId import com.normation.rudder.git.GitArchiveId import com.normation.rudder.git.GitPath @@ -99,7 +97,6 @@ import com.normation.rudder.repository.ldap.WoLDAPRuleRepository import com.normation.rudder.repository.ldap.ZioTReentrantLock import com.normation.rudder.repository.xml.GitParseTechniqueLibrary import com.normation.rudder.services.eventlog.EventLogFactory -import com.normation.rudder.services.nodes.LDAPNodeInfo import com.normation.rudder.services.nodes.NodeInfoService import com.normation.rudder.services.policies.NodeConfigData import com.normation.rudder.services.policies.TechniqueAcceptationUpdater @@ -164,12 +161,12 @@ class TestMigrateSystemTechniques7_0 extends Specification { val nodeInfoService = new NodeInfoService { override def getAll(): IOResult[Map[NodeId, NodeInfo]] = List(root, relay1).map(x => (x.id, x)).toMap.succeed - override def getLDAPNodeInfo(nodeIds: Set[NodeId], predicates: Seq[NodeInfoMatcher], composition: CriterionComposition): IOResult[Set[LDAPNodeInfo]] = ??? override def getNodeInfo(nodeId: NodeId): IOResult[Option[NodeInfo]] = ??? override def getNodeInfos(nodeIds: Set[NodeId]): IOResult[Set[NodeInfo]] = ??? override def getNumberOfManagedNodes: Int = ??? override def getAllNodesIds(): IOResult[Set[NodeId]] = ??? override def getAllNodes(): IOResult[Map[NodeId, Node]] = ??? + override def getAllNodeInfos(): IOResult[Seq[NodeInfo]] = ??? override def getAllSystemNodeIds(): IOResult[Seq[NodeId]] = ??? override def getPendingNodeInfos(): IOResult[Map[NodeId, NodeInfo]] = ??? override def getPendingNodeInfo(nodeId: NodeId): IOResult[Option[NodeInfo]] = ???