From 61c5af46ec0842e832a29c7247178f7f3ff1e5f2 Mon Sep 17 00:00:00 2001 From: "Francois @fanf42 Armand" Date: Wed, 28 Mar 2018 19:01:56 +0200 Subject: [PATCH] Work in progress --- .../services/queries/DynGroupService.scala | 149 +++++++++++++++--- 1 file changed, 127 insertions(+), 22 deletions(-) diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupService.scala b/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupService.scala index 0ecd6b71e48..1d4de5f128b 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupService.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/queries/DynGroupService.scala @@ -38,13 +38,16 @@ package com.normation.rudder.services.queries import com.normation.inventory.domain.NodeId -import com.normation.rudder.domain.nodes.{NodeGroup,NodeGroupId} +import com.normation.rudder.domain.nodes.{NodeGroup, NodeGroupId} import com.normation.ldap.sdk._ import BuildFilter._ -import com.normation.rudder.domain.{RudderDit,RudderLDAPConstants} +import com.normation.rudder.domain.{RudderDit, RudderLDAPConstants} import RudderLDAPConstants._ import com.normation.utils.Control.sequence import com.normation.inventory.ldap.core.LDAPConstants +import com.normation.rudder.domain.queries.CriterionLine +import com.normation.rudder.domain.queries.ExactStringComparator +import com.normation.rudder.domain.queries.Query import com.normation.rudder.repository.ldap.LDAPEntityMapper import net.liftweb.common._ @@ -119,41 +122,143 @@ class DynGroupServiceImpl( * If any error is encountered during the sequence of tests, * the whole process is in error. */ - override def findDynGroups(nodeIds:Seq[NodeId]) : Box[Map[NodeId,Seq[NodeGroupId]]] = { + override def findDynGroups(nodeIds:Set[NodeId]) : Box[Map[NodeId, Seq[NodeGroupId]]] = { + + for { - con <- ldap - dyngroups <- sequence(con.searchSub(rudderDit.GROUP.dn, dynGroupFilter, dynGroupAttrs:_*)) { entry => - mapper.entry2NodeGroup(entry) ?~! "Can not map entry to a node group: %s".format(entry) - } + con <- ldap + dyngroups <- sequence(con.searchSub(rudderDit.GROUP.dn, dynGroupFilter, dynGroupAttrs:_*)) { entry => + mapper.entry2NodeGroup(entry) ?~! "Can not map entry to a node group: %s".format(entry) + } //now, for each test the query - mapGroupAndNodes <- sequence(dyngroups) { g => - (for { - matchedIds <- g match { - case NodeGroup(id, _, _, Some(query), true, _, _, _) => - queryChecker.check(query,nodeIds) - case g => { //what ? - logger.error("Found a group without a query or not dynamic: %s".format(g)) - Full(Seq()) + mapGroupAndNodes <- processDynGroups(dyngroups.toList, nodeIds) + } yield { + swapMap(mapGroupAndNodes) + } + } + + /** + * Given a list of dynamique groups and a list of nodes, find which groups contains which + * nodes. + * This method proceed recursively, starting by dealing with nodes without any other group + * dependency (ie: group of groups are set aside), and then continuing until everything + * is done. See https://www.rudder-project.org/redmine/issues/12060#note-6 for + * algo detail in image. + * + * Raw behavior: 3 queues: TODO, BLOCKED, DONE + * All dyn groups start in TODO with the list of nodeIds and their query splitted in two parts: + * - a list of depandent groups + * - simple query criterions + * + * Init: all groups with at least one dependency go to BLOCKED + * Then iterativelly proceed groups in TODO (until TODO empty) so that for each group G: + * - compute nodes in G + * - put G in DONE + * - find all node in BLOCKED with a dependency toward G, and for each of these H group, do: + * - union or intersect G nodes with H groups (depending of the H composition kind) + * - remove G as dependency from H + * - if H has no more dependencies, put it back at the end of TODO + * When TODO is empty, do: + * - for each remaining groups in BLOCKED, act as if there dependencies have 0 nodes (and => empty, or => keep + * only simple query part) + * - put empty group into DONE, and non-empty one into TODO + * - process TODO + * + * And done ! :) + */ + def processDynGroups(groups: List[NodeGroup], nodeIds: Set[NodeId]): Box[List[(NodeGroupId, Set[NodeId])]] = { + // a data structure to keep a group ID, set of nodes, dependencies, query and composition/ + // the query does not contain group anymore. + final case class DynGroup(id: NodeGroupId, nodeIds: Set[NodeId], query: Query, dependencies: Set[NodeGroupId]) + + /* + * one step of the algo + */ + def recProcess(todo: List[DynGroup], blocked: List[DynGroup], done: List[(NodeGroupId, Set[NodeId])]): Box[List[(NodeGroupId, Set[NodeId])]] = { + import com.normation.rudder.domain.queries.{ And => CAnd} + + (todo, blocked, done) match { + + case (Nil, Nil, res) => // termination condition + Full(res) + + case (Nil, b, res) => // end of main phase: zero-ïze group dependencies in b and put them back in the other two queues + val (newTodo, newRes) = ( (List.empty[DynGroup], res) /: b) { case ( (t, r), next ) => + if(next.query.composition == CAnd) { // the group has zero node b/c intersect with 0 => new result + (t, (next.id, Set.empty[NodeId])::r ) + } else { // we can just ignore the dependencies and proceed the remaining group as a normal dyn group + (next :: t, r) } } - } yield { - (g.id, matchedIds) - }) ?~! "Error when trying to find what nodes belong to dynamic group %s".format(g) + // start back the process with empty BLOCKED + recProcess(newTodo, Nil, newRes) + + + case (h::tail, b, res) => // standard step: takes the group and deals with it + (queryChecker.check(h.query, h.nodeIds.toSeq).flatMap { nIds => + val setNodeIds = nIds.toSet + // get blocked group with h as a dep + val (withDep, stillBlocked) = b.partition( _.dependencies.contains(h.id) ) + // for each node with that dep: intersect or union nodeids, remove dep + val newTodos = withDep.map { case DynGroup(id, nodes, query, dependencies) => + val newNodeIds = if(query.composition == CAnd) { + nodes.intersect(setNodeIds) + } else { + nodes.union(setNodeIds) + } + val newDep = dependencies - h.id + DynGroup(id, newNodeIds, query, newDep) + } + recProcess(tail ::: newTodos, stillBlocked, (h.id, setNodeIds) :: res) + }) ?~! s"Error when trying to find what nodes belong to dynamic group ${h.id}" + } + } + + // init: transform NodeGroups into DynGroup. + // group without query are filtered out (they will have 0 node so that does not change + // anything if a group had a dep towards one of them) + val dynGroups = groups.flatMap { g => + if(g._isEnabled) { + g.query.map { query => + // partition group query into Subgroup / simple criterion + val (dep, criteria) = ( (Set.empty[NodeGroupId], List.empty[CriterionLine])/: query.criteria) { case ( (g, q), next) => + if(next.objectType.objectType == "group") { // it's a dependency + // we only know how to process the comparator "exact string match" for group + if(next.comparator == ExactStringComparator) { + ( g + NodeGroupId(next.value), q) + } else { + logger.warn("Warning: group criteria use something else than exact string match comparator: " + next) + (g, q) + } + } else { // keep that criterium + (g, next :: q) + } + } + DynGroup(g.id, nodeIds, query.copy(criteria = criteria), dep) + } + } else { + None } - } yield { - swapMap(mapGroupAndNodes) } + + + // partition nodes with dependencies / node without: + val (nodep, withdep) = dynGroups.partition( _.dependencies.isEmpty ) + + // start the process ! End at the end, transform the result into a map. + recProcess(nodep, withdep, Nil) } + /** * Transform the map of (groupid => seq(nodeids) into a map of * (nodeid => seq(groupids) */ - private[this] def swapMap(source:Seq[(NodeGroupId,Seq[NodeId])]) : Map[NodeId,Seq[NodeGroupId]] = { + private[this] def swapMap(source:Seq[(NodeGroupId, Set[NodeId])]) : Map[NodeId, Seq[NodeGroupId]] = { val dest = scala.collection.mutable.Map[NodeId,List[NodeGroupId]]() for { (gid, seqNodeIds) <- source - nodeId <- seqNodeIds + nodeId <- seqNodeIds } { dest(nodeId) = gid :: dest.getOrElse(nodeId,Nil) }