Fixes #24562: When moving an inventory to \"received\", check if othe…

…r file for same node uuid exists and delete it

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/NodeGroupRepository.scala

@@ -207,6 +207,8 @@ trait RoNodeGroupRepository {
    */
   def getFullGroupLibrary(): IOResult[FullNodeGroupCategory]
 
+  def categoryExists(id: NodeGroupCategoryId): IOResult[Boolean]
+
   /**
    * Get a server group by its id. Fail if not present.
    * @param id

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala

@@ -282,6 +282,13 @@ class RoLDAPNodeGroupRepository(
     else getParentGroupCategory(id).flatMap(parent => getParentsForCategory(parent.id, root).map(parents => parent :: parents))
   }
 
+  override def categoryExists(id: NodeGroupCategoryId): IOResult[Boolean] = {
+    for {
+      con           <- ldap
+      categoryEntry <- groupLibMutex.readLock(getCategoryEntry(con, id, "dn"))
+    } yield categoryEntry.nonEmpty
+  }
+
   /**
    * Get a group category by its id
    * */

webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala

@@ -64,6 +64,7 @@ import com.normation.rudder.domain.appconfig.FeatureSwitch
 import com.normation.rudder.domain.logger.ApplicationLogger
 import com.normation.rudder.domain.logger.ApplicationLoggerPure
 import com.normation.rudder.domain.nodes.NodeGroup
+import com.normation.rudder.domain.nodes.NodeGroupCategory
 import com.normation.rudder.domain.nodes.NodeGroupCategoryId
 import com.normation.rudder.domain.nodes.NodeGroupId
 import com.normation.rudder.domain.policies.Directive
@@ -404,6 +405,17 @@ object JGroupCategory {
   }
 }
 
+object ZipArchiveBuilderService {
+  // names of directories under the root directory of the archive
+  val RULES_DIR      = "rules"
+  val GROUPS_DIR     = "groups"
+  val DIRECTIVES_DIR = "directives"
+  val TECHNIQUES_DIR = "techniques"
+
+  val GROUP_CAT_FILENAME = "category.json"
+  val RULE_CATS          = "rule-categories.json"
+}
+
 /**
  * That class is in charge of building a archive of a set of rudder objects.
  * It knows how to get objects from their ID, serialise them to the expected
@@ -422,14 +434,7 @@ class ZipArchiveBuilderService(
     groupRepo:              RoNodeGroupRepository
 ) {
 
-  // names of directories under the root directory of the archive
-  val RULES_DIR      = "rules"
-  val GROUPS_DIR     = "groups"
-  val DIRECTIVES_DIR = "directives"
-  val TECHNIQUES_DIR = "techniques"
-
-  val GROUP_CAT_FILENAME = "category.json"
-  val RULE_CATS          = "rule-categories.json"
+  import ZipArchiveBuilderService.*
 
   /*
    * get the content of the JSON string in the format expected by Zippable
@@ -459,12 +464,18 @@ class ZipArchiveBuilderService(
     val name = fileArchiveNameService.toFileName(origName) + extension
 
     // find a free name, avoiding overwriting a previous similar one
-    usedNames.modify(m => {
-      val realName = if (m(category).contains(name)) {
-        findRecName(m(category), name, 1)
-      } else name
-      (realName, m + ((category, m(category) + realName)))
-    })
+    usedNames.modify { m =>
+      m.get(category) match {
+        case None                           =>
+          (name, m + ((category, Set(name))))
+        case Some(c) if (!c.contains(name)) =>
+          (name, m + ((category, c + name)))
+        case Some(c)                        =>
+          val realName = findRecName(c, name, 1)
+          (realName, m + ((category, m(category) + realName)))
+      }
+
+    }
   }
 
   /*
@@ -527,7 +538,7 @@ class ZipArchiveBuilderService(
                     }
                     .reverse ++ contents.map { case (p, opt) => Zippable.make(basePath + p, opt) }
       _        <- ApplicationLoggerPure.Archive.debug(
-                    s"Building archive '${archiveName}': adding technique zipables: ${zips.map(_.path).mkString(", ")}"
+                    s"Building archive '${archiveName}': adding technique zippables: ${zips.map(_.path).mkString(", ")}"
                   )
     } yield {
       zips
@@ -547,17 +558,18 @@ class ZipArchiveBuilderService(
   ): IOResult[Seq[Zippable]] = {
 
     import com.softwaremill.quicklens.*
+
     // filter the group category, keeping only groups from list, then non-empty cat, recursively
     def recFilter(ids: Seq[NodeGroupId], cat: FullNodeGroupCategory): Option[FullNodeGroupCategory] = {
       if (ids.isEmpty) None
       else {
         val subCats   = cat.subCategories.flatMap(recFilter(ids, _))
-        val subGroups = cat.targetInfos.filter(t => {
+        val subGroups = cat.targetInfos.filter { t =>
           t.target.target match {
             case GroupTarget(id) => ids.contains(id)
             case _               => false
           }
-        })
+        }
 
         if (subCats.isEmpty && subGroups.isEmpty) None
         else {
@@ -600,7 +612,7 @@ class ZipArchiveBuilderService(
       for {
         ref   <- Ref.make(Chunk.empty[Zippable])
         _     <- ZIO.foreach(cat.subCategories) { sub =>
-                   val c    = JGroupCategory.fromFullNodeGroupCategory(cat)
+                   val c    = JGroupCategory.fromFullNodeGroupCategory(sub)
                    val json = c.toJsonPretty
                    for {
                      catDir     <- findName(c.name, "", usedNames, basePath)
@@ -609,7 +621,7 @@ class ZipArchiveBuilderService(
                      catFilePath = path + "/" + catName
                      _          <- ApplicationLoggerPure.Archive
                                      .debug(s"Building archive '${rootDirName}': adding group category zippable: ${catFilePath}")
-                     zipHead     = Chunk(Zippable(catFilePath, Some(getJsonZippableContent(json))))
+                     _          <- ref.update(_.appended(Zippable(catFilePath, Some(getJsonZippableContent(json)))))
                      children   <- recMapToZippable(path, sub, catFileName, usedNames)
                      _          <- ref.update(_.appendedAll(children))
                    } yield ()
@@ -630,13 +642,16 @@ class ZipArchiveBuilderService(
       } yield zips
     }
 
+    // we need the file name without extension
+    val GROUP_CAT_WITHOUT_EXT = GROUP_CAT_FILENAME.split("\\.")(0)
+
     recFilter(ids, groupLib) match {
       case None      => if (ids.isEmpty) Nil.succeed else missingGroups(ids)
       case Some(cat) =>
-        val missing = cat.allGroups.keySet.intersect(ids.toSet)
+        val missing = cat.allGroups.keySet -- ids
         if (missing.nonEmpty) missingGroups(missing)
         else {
-          recMapToZippable(groupsDir, cat, GROUP_CAT_FILENAME, usedNames)
+          recMapToZippable(groupsDir, cat, GROUP_CAT_WITHOUT_EXT, usedNames)
         }
     }
   }
@@ -706,7 +721,8 @@ class ZipArchiveBuilderService(
       groupsDirZip     = Zippable(groupsDir, None)
       depGroups       <- if (includeDepGroups) depGroupIds.get else Nil.succeed
       groupLib        <- groupRepo.getFullGroupLibrary()
-      groupsZip       <- getGroupLibZippable(groupIds ++ depGroups, GROUPS_DIR, usedNames, rootDirName, groupLib)
+      groupDir         = root + "/" + GROUPS_DIR
+      groupsZip       <- getGroupLibZippable(groupIds ++ depGroups, groupDir, usedNames, rootDirName, groupLib)
       // directives need access to technique, but we don't want to look up several time the same one
       techniques      <- Ref.Synchronized.make(Map.empty[TechniqueId, (Chunk[TechniqueCategoryName], Technique)])
 
@@ -783,6 +799,11 @@ final case class DirectiveArchive(
     technique: TechniqueName,
     directive: Directive
 )
+final case class GroupCategoryArchive(
+    category: JGroupCategory,
+    // the path for that cat
+    catPath:  String
+)
 final case class GroupArchive(
     group:    NodeGroup,
     category: NodeGroupCategoryId
@@ -796,29 +817,33 @@ final case class PolicyArchive(
     metadata:   PolicyArchiveMetadata,
     techniques: Chunk[TechniqueArchive],
     directives: Chunk[DirectiveArchive],
+    groupCats:  Chunk[GroupCategoryArchive],
     groups:     Chunk[GroupArchive],
     rules:      Chunk[Rule]
 ) {
   def debugString: String = {
     s"""Archive ${metadata.filename}:
-       | - techniques: ${techniques.map(_.technique.id.serialize).sorted.mkString(", ")}
-       | - directives: ${directives.map(d => s"'${d.directive.name}' [${d.directive.id.serialize}]").sorted.mkString(", ")}
-       | - groups    : ${groups.map(d => s"'${d.group.name}' [${d.group.id.serialize}]").sorted.mkString(", ")}
-       | - rules     : ${rules.map(d => s"'${d.name}' [${d.id.serialize}]").sorted.mkString(", ")}""".stripMargin
+       | - techniques      : ${techniques.map(_.technique.id.serialize).sorted.mkString(", ")}
+       | - directives      : ${directives.map(d => s"'${d.directive.name}' [${d.directive.id.serialize}]").sorted.mkString(", ")}
+       | - group categories: ${groupCats.map(c => s"'${c.category.name}' [${c.category.id}]").sorted.mkString(", ")}
+       | - groups          : ${groups.map(g => s"'${g.group.name}' [${g.group.id.serialize}]").sorted.mkString(", ")}
+       | - rules           : ${rules.map(r => s"'${r.name}' [${r.id.serialize}]").sorted.mkString(", ")}""".stripMargin
   }
 }
 object PolicyArchive {
-  def empty: PolicyArchive = PolicyArchive(PolicyArchiveMetadata.empty, Chunk.empty, Chunk.empty, Chunk.empty, Chunk.empty)
+  def empty: PolicyArchive =
+    PolicyArchive(PolicyArchiveMetadata.empty, Chunk.empty, Chunk.empty, Chunk.empty, Chunk.empty, Chunk.empty)
 }
 
 final case class SortedEntries(
     techniques: Chunk[(String, Array[Byte])],
     directives: Chunk[(String, Array[Byte])],
+    groupCats:  Chunk[(String, Array[Byte])],
     groups:     Chunk[(String, Array[Byte])],
     rules:      Chunk[(String, Array[Byte])]
 )
 object SortedEntries {
-  def empty: SortedEntries = SortedEntries(Chunk.empty, Chunk.empty, Chunk.empty, Chunk.empty)
+  def empty: SortedEntries = SortedEntries(Chunk.empty, Chunk.empty, Chunk.empty, Chunk.empty, Chunk.empty)
 }
 
 final case class PolicyArchiveUnzip(
@@ -850,6 +875,7 @@ class ZipArchiveReaderImpl(
   val techniqueRegex: Regex = """(.*/|)techniques/(.+)""".r
   val metadataRegex:  Regex = """(.+)/metadata.xml""".r
   val directiveRegex: Regex = """(.*/|)directives/(.+.json)""".r
+  val groupCatsRegex: Regex = """(.*/|)groups/(.*category.json)""".r
   val groupRegex:     Regex = """(.*/|)groups/(.+.json)""".r
   val ruleRegex:      Regex = """(.*/|)rules/(.+.json)""".r
 
@@ -913,6 +939,15 @@ class ZipArchiveReaderImpl(
       .toIO
       .flatMap(_.toDirective().map(t => DirectiveArchive(t._1, t._2)))
   }
+  def parseGroupCat(name: String, content: Array[Byte])(implicit
+      dec: JsonDecoder[JGroupCategory]
+  ): IOResult[GroupCategoryArchive] = {
+    // we need to keep path to be able to find back hierarchy later on
+    (new String(content, StandardCharsets.UTF_8))
+      .fromJson[JGroupCategory]
+      .toIO
+      .map(c => GroupCategoryArchive(c, File(name).parent.pathAsString))
+  }
   def parseGroup(name: String, content: Array[Byte])(implicit dec: JsonDecoder[JRGroup]):         IOResult[GroupArchive]     = {
     (new String(content, StandardCharsets.UTF_8))
       .fromJson[JRGroup]
@@ -959,6 +994,11 @@ class ZipArchiveReaderImpl(
   ): IOResult[PolicyArchiveUnzip] = {
     parseSimpleFile(arch, directives, modifyLens[PolicyArchiveUnzip](_.policies.directives), parseDirective)
   }
+  def parseGroupCats(arch: PolicyArchiveUnzip, cats: Chunk[(String, Array[Byte])])(implicit
+      dec: JsonDecoder[JGroupCategory]
+  ): IOResult[PolicyArchiveUnzip] = {
+    parseSimpleFile(arch, cats, modifyLens[PolicyArchiveUnzip](_.policies.groupCats), parseGroupCat)
+  }
   def parseGroups(arch: PolicyArchiveUnzip, groups: Chunk[(String, Array[Byte])])(implicit
       dec: JsonDecoder[JRGroup]
   ): IOResult[PolicyArchiveUnzip] = {
@@ -982,6 +1022,9 @@ class ZipArchiveReaderImpl(
           case (directiveRegex(_, x), Some(content)) =>
             ApplicationLoggerPure.Archive.logEffect.trace(s"Archive '${archiveName}': found directive file ${x}")
             arch.modify(_.directives).using(_ :+ (x, content))
+          case (groupCatsRegex(_, x), Some(content)) =>
+            ApplicationLoggerPure.Archive.logEffect.trace(s"Archive '${archiveName}': found group category file ${x}")
+            arch.modify(_.groupCats).using(_ :+ (x, content))
           case (groupRegex(_, x), Some(content))     =>
             ApplicationLoggerPure.Archive.logEffect.trace(s"Archive '${archiveName}': found group file ${x}")
             arch.modify(_.groups).using(_ :+ (x, content))
@@ -1032,7 +1075,11 @@ class ZipArchiveReaderImpl(
       _              <- ApplicationLoggerPure.Archive.debug(
                           s"Processing archive '${archiveName}': groups: '${sortedEntries.groups.map(_._1).mkString("', '")}'"
                         )
-      withGroups     <- parseGroups(withDirectives, sortedEntries.groups)
+      withGroupCats  <- parseGroupCats(withDirectives, sortedEntries.groupCats)
+      _              <- ApplicationLoggerPure.Archive.debug(
+                          s"Processing archive '${archiveName}': rules: '${sortedEntries.rules.map(_._1).mkString("', '")}'"
+                        )
+      withGroups     <- parseGroups(withGroupCats, sortedEntries.groups)
       _              <- ApplicationLoggerPure.Archive.debug(
                           s"Processing archive '${archiveName}': rules: '${sortedEntries.rules.map(_._1).mkString("', '")}'"
                         )
@@ -1141,6 +1188,8 @@ class SaveArchiveServicebyRepo(
     asyncDeploy:       AsyncDeploymentActor
 ) extends SaveArchiveService {
 
+  val GroupRootId = NodeGroupCategoryId("GroupRoot")
+
   /*
    * Saving a techniques:
    * - override all files that are coming from archive
@@ -1229,13 +1278,65 @@ class SaveArchiveServicebyRepo(
       _  <- woDirectiveRepos.saveDirective(at.id, d.directive, eventMetadata.modId, eventMetadata.actor, eventMetadata.msg)
     } yield ()
   }
+  def saveGroupCat(
+      eventMetadata: EventMetadata,
+      cat:           GroupCategoryArchive,
+      catMap:        Map[String, NodeGroupCategoryId]
+  ): IOResult[Unit] = {
+    for {
+      _       <- ApplicationLoggerPure.Archive.debug(s"Adding group category from archive: '${cat.category.name}' (${cat.category.id})")
+      /*
+       * Categories are sorted by parents-first in the archive, so we are sure at that
+       * point that at least all parents exists. But perhaps not that one.
+       */
+      id       = NodeGroupCategoryId(cat.category.id)
+      exists  <- roGroupRepos.categoryExists(id)
+      parentId = catMap.get(File(cat.catPath).parent.pathAsString) match {
+                   case Some(p) => p
+                   case None    => GroupRootId
+                 }
+      category = NodeGroupCategory(id, cat.category.name, cat.category.description, Nil, Nil)
+      _       <- if (exists) {
+                   woGroupRepos.saveGroupCategory(category, parentId, eventMetadata.modId, eventMetadata.actor, eventMetadata.msg)
+                 } else {
+                   woGroupRepos.addGroupCategorytoCategory(
+                     category,
+                     parentId,
+                     eventMetadata.modId,
+                     eventMetadata.actor,
+                     eventMetadata.msg
+                   )
+                 }
+    } yield ()
+  }
   def saveGroup(eventMetadata: EventMetadata, g: GroupArchive):                  IOResult[Unit] = {
     for {
       _ <- ApplicationLoggerPure.Archive.debug(s"Adding group from archive: '${g.group.name}' (${g.group.id.serialize})")
+      // normally, at that point the category of the group exists because we took care to create them before.
+      // But we used to have a bug where categories where not exported, so until Rudder 9 we need to check for that
+      // if category of group doesn't exist, we need to create one using the UUID as name
+      c <- roGroupRepos.categoryExists(g.category)
+      _ <- ZIO.when(!c) {
+             woGroupRepos.addGroupCategorytoCategory(
+               NodeGroupCategory(g.category, g.category.value, "", Nil, Nil),
+               GroupRootId,
+               eventMetadata.modId,
+               eventMetadata.actor,
+               eventMetadata.msg
+             )
+           }
+      // now we can check if we create a new group or update one
       x <- roGroupRepos.getNodeGroupOpt(g.group.id)
       _ <- x match {
-             case Some(value) => woGroupRepos.update(g.group, eventMetadata.modId, eventMetadata.actor, eventMetadata.msg)
-             case None        => woGroupRepos.create(g.group, g.category, eventMetadata.modId, eventMetadata.actor, eventMetadata.msg)
+             case Some((_, parentId)) =>
+               woGroupRepos.update(g.group, eventMetadata.modId, eventMetadata.actor, eventMetadata.msg) *>
+               // we need to check if the group moved
+               ZIO.when(parentId != g.category) {
+                 woGroupRepos.move(g.group.id, g.category, eventMetadata.modId, eventMetadata.actor, eventMetadata.msg)
+               }
+
+             case None =>
+               woGroupRepos.create(g.group, g.category, eventMetadata.modId, eventMetadata.actor, eventMetadata.msg)
            }
     } yield ()
   }
@@ -1266,6 +1367,9 @@ class SaveArchiveServicebyRepo(
       _ <- ZIO.foreach(archive.techniques)(saveTechnique(eventMetadata, _))
       _ <- IOResult.attempt(techniqueReader.readTechniques)
       _ <- ZIO.foreach(archive.directives)(saveDirective(eventMetadata, _))
+      // we need the map of all (categoryPath -> id) in the archive
+      m  = archive.groupCats.map(c => (c.catPath, NodeGroupCategoryId(c.category.id))).toMap
+      _ <- ZIO.foreach(archive.groupCats)(saveGroupCat(eventMetadata, _, m))
       _ <- ZIO.foreach(archive.groups)(saveGroup(eventMetadata, _))
       _ <- ZIO.foreach(archive.rules)(saveRule(eventMetadata, mergePolicy, _))
       // update technique lib, regenerate policies

webapp/sources/rudder/rudder-rest/src/test/resources/api/api_groups.yml

@@ -15,7 +15,7 @@ response:
             "id":"0000f5d3-8c61-4d20-88a7-bb947705ba8a",
             "displayName":"Real nodes",
             "description":"",
-            "category":"GroupRoot",
+            "category":"category1",
             "query":{
               "select":"nodeAndPolicyServer",
               "composition":"or",
@@ -245,7 +245,7 @@ response:
             "id":"0000f5d3-8c61-4d20-88a7-bb947705ba8a",
             "displayName":"Real nodes",
             "description":"",
-            "category":"GroupRoot",
+            "category":"category1",
             "query":{
               "select":"nodeAndPolicyServer",
               "composition":"or",
@@ -1329,7 +1329,7 @@ response:
             "219b9c98-3d1e-44c9-aff5-95b4fc7c4ada"
           ],
           "groups":[
-
+            "0000f5d3-8c61-4d20-88a7-bb947705ba8a"
           ],
           "targets":[]
         }