Fixes #24789: Archive API for import/export doesn't know about technique

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/domain/TechniqueCategory.scala

@@ -37,7 +37,11 @@
 
 package com.normation.cfclerk.domain
 
+import com.normation.rudder.domain.policies.ActiveTechniqueCategory
+import com.normation.rudder.domain.policies.ActiveTechniqueCategoryId
 import scala.collection.SortedSet
+import scala.xml.Elem
+import zio.json.*
 
 /**
  * A policy category name.
@@ -47,6 +51,53 @@ import scala.collection.SortedSet
  */
 final case class TechniqueCategoryName(value: String) extends AnyVal
 
+/*
+ * Just the name / description of a technique category without all the
+ * parent / subcategories / techniques stuff.
+ */
+final case class TechniqueCategoryMetadata(name: String, description: String, isSystem: Boolean)
+
+object TechniqueCategoryMetadata {
+  implicit val codecTechniqueCategoryMetadata: JsonCodec[TechniqueCategoryMetadata] = DeriveJsonCodec.gen
+
+  implicit class ToActiveTechniqueCategory(metadata: TechniqueCategoryMetadata) {
+    def toActiveTechniqueCategory(id: ActiveTechniqueCategoryId): ActiveTechniqueCategory = ActiveTechniqueCategory(
+      id,
+      metadata.name,
+      metadata.description,
+      Nil,
+      Nil
+    )
+
+    def toXml: Elem = {
+      <xml>
+        <name>{metadata.name}</name>
+        <description>{metadata.description}</description>
+        {if (metadata.isSystem) <system>true</system> else xml.NodeSeq.Empty}
+      </xml>
+    }
+  }
+
+  def parseXML(xml: Elem, defaultName: String): TechniqueCategoryMetadata = {
+    def nonEmpty(s: String): Option[String] = {
+      s match {
+        case null | "" => None
+        case _         => Some(s)
+      }
+    }
+
+    val name        = nonEmpty((xml \\ "name").text).getOrElse(defaultName)
+    val description = nonEmpty((xml \\ "description").text).getOrElse("")
+    val isSystem    = (nonEmpty((xml \\ "system").text).getOrElse("false")).equalsIgnoreCase("true")
+
+    TechniqueCategoryMetadata(name, description, isSystem = isSystem)
+  }
+
+  // the default file name for category metadata.
+  val FILE_NAME_XML  = "category.xml"
+  val FILE_NAME_JSON = "category.json"
+}
+
 sealed abstract class TechniqueCategoryId(val name: TechniqueCategoryName) {
 
   /**

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala

@@ -876,26 +876,17 @@ class GitTechniqueReader(
       parseDescriptor:    Boolean // that option is a success optimization for the case diff between old/new commit
   ): IOResult[TechniqueCategory] = {
 
-    def nonEmpty(s: String):                                                   Option[String]                      = {
-      s match {
-        case null | "" => None
-        case _         => Some(s)
-      }
-    }
-    def parse(db: Repository, parseDesc: Boolean, catId: TechniqueCategoryId): IOResult[(String, String, Boolean)] = {
+    def parse(db: Repository, parseDesc: Boolean, catId: TechniqueCategoryId): IOResult[TechniqueCategoryMetadata] = {
       if (parseDesc) {
         val managedStream =
           ZIO.acquireRelease(IOResult.attempt(db.open(descriptorObjectId).openStream))(is => effectUioUnit(is.close()))
         for {
           xml <- loadDescriptorFile(managedStream, filePath)
         } yield {
-          val name        = nonEmpty((xml \\ "name").text).getOrElse(catId.name.value)
-          val description = nonEmpty((xml \\ "description").text).getOrElse("")
-          val isSystem    = (nonEmpty((xml \\ "system").text).getOrElse("false")).equalsIgnoreCase("true")
-          (name, description, isSystem)
+          TechniqueCategoryMetadata.parseXML(xml, catId.name.value)
         }
       } else {
-        (catId.name.value, "", false).succeed
+        TechniqueCategoryMetadata(catId.name.value, "", false).succeed
       }
     }
 
@@ -906,7 +897,7 @@ class GitTechniqueReader(
     for {
       triple <- parse(db, parseDescriptor, catId)
     } yield {
-      val (name, desc, system) = triple
+      val TechniqueCategoryMetadata(name, desc, system) = triple
       catId match {
         case RootTechniqueCategoryId => RootTechniqueCategory(name, desc, isSystem = system)
         case sId: SubTechniqueCategoryId => SubTechniqueCategory(sId, name, desc, isSystem = system)

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/git/ZipUtils.scala

@@ -136,7 +136,11 @@ object ZipUtils {
   def zip(zipout: OutputStream, toAdds: Seq[Zippable]): IOResult[Unit] = {
     // we must ensure that each entry is unique, else zip fails
     val unique = toAdds.distinctBy(_.path)
-    ZIO.acquireReleaseWith(IOResult.attempt(new ZipOutputStream(zipout)))(zout => effectUioUnit(zout.close())) { zout =>
+    ZIO.acquireReleaseWith(IOResult.attempt(new ZipOutputStream(zipout)))(zout => {
+      // if the connection is interrupted, for ex if you use curl without a --output arg,
+      // then the usual effectUioUnit(zout.close()) leads to a big stack trace (unactionnable, uninteresting).
+      ZIO.attemptBlocking(zout.close()).orElseSucceed(ZIO.unit)
+    }) { zout =>
       val addToZout = (is: InputStream) => IOResult.attempt("Error when copying file")(IOUtils.copy(is, zout))
 
       ZIO.foreachDiscard(unique) { x =>

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ItemArchiveManager.scala

@@ -339,6 +339,8 @@ trait GitActiveTechniqueCategoryArchiver {
    * managed by git.
    * If gitCommit is true, the modification is
    * saved in git. Else, no modification in git are saved.
+   *
+   * Only the metadata part (id, description...) are save ; item and children are ignored.
    */
   def archiveActiveTechniqueCategory(
       uptc:       ActiveTechniqueCategory,

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitArchivers.scala

@@ -40,6 +40,7 @@ package com.normation.rudder.repository.xml
 import com.normation.NamedZioLogger
 import com.normation.cfclerk.domain.SectionSpec
 import com.normation.cfclerk.domain.Technique
+import com.normation.cfclerk.domain.TechniqueCategoryMetadata
 import com.normation.cfclerk.domain.TechniqueId
 import com.normation.cfclerk.domain.TechniqueName
 import com.normation.cfclerk.services.TechniqueRepository
@@ -52,6 +53,7 @@ import com.normation.rudder.domain.Constants.CONFIGURATION_RULES_ARCHIVE_TAG
 import com.normation.rudder.domain.Constants.GROUPS_ARCHIVE_TAG
 import com.normation.rudder.domain.Constants.PARAMETERS_ARCHIVE_TAG
 import com.normation.rudder.domain.Constants.POLICY_LIBRARY_ARCHIVE_TAG
+import com.normation.rudder.domain.logger.GitArchiveLoggerPure
 import com.normation.rudder.domain.nodes.NodeGroup
 import com.normation.rudder.domain.nodes.NodeGroupCategory
 import com.normation.rudder.domain.nodes.NodeGroupCategoryId
@@ -69,6 +71,7 @@ import com.normation.rudder.repository.*
 import com.normation.rudder.services.marshalling.*
 import com.normation.rudder.services.user.PersonIdentService
 import java.io.File
+import java.io.FileNotFoundException
 import net.liftweb.common.*
 import org.apache.commons.io.FileUtils
 import org.eclipse.jgit.lib.PersonIdent
@@ -237,6 +240,7 @@ trait TechniqueArchiver {
       committer:   EventActor,
       msg:         String
   ): IOResult[Unit]
+
   def saveTechnique(
       techniqueId:     TechniqueId,
       categories:      Seq[String],
@@ -245,6 +249,14 @@ trait TechniqueArchiver {
       committer:       EventActor,
       msg:             String
   ): IOResult[Unit]
+
+  def saveTechniqueCategory(
+      categories: Seq[String], // path (inclusive) to the category
+      metadata:   TechniqueCategoryMetadata,
+      modId:      ModificationId,
+      committer:  EventActor,
+      msg:        String
+  ): IOResult[Unit]
 }
 
 /*
@@ -361,6 +373,40 @@ class TechniqueArchiverImpl(
     } yield ()).chainError(s"error when committing Technique '${techniqueId.serialize}'").unit
   }
 
+  def saveTechniqueCategory(
+      categories: Seq[String], // path (inclusive) to the category
+      metadata:   TechniqueCategoryMetadata,
+      modId:      ModificationId,
+      committer:  EventActor,
+      msg:        String
+  ): IOResult[Unit] = {
+    val categoryPath = categories.filter(_ != "/").mkString("/")
+    val catGitPath   = s"${relativePath}/${categoryPath}/${TechniqueCategoryMetadata.FILE_NAME_XML}"
+    val categoryFile = gitRepo.rootDirectory / catGitPath
+    val xml          = metadata.toXml
+
+    categories.lastOption match {
+      case None        => Unexpected("You can't change the root category information").fail
+      case Some(catId) =>
+        (for {
+          // the file may not exist, which is not an error in that case
+          existing <- IOResult.attempt {
+                        val elem = XML.load(Source.fromFile(categoryFile.toJava))
+                        Some(TechniqueCategoryMetadata.parseXML(elem, catId))
+                      }.catchSome { case SystemError(_, _: FileNotFoundException) => None.succeed }
+          _        <- if (existing.contains(metadata)) {
+                        GitArchiveLoggerPure.debug(s"Not commiting '${catGitPath}' because it already exists with these values")
+                      } else {
+                        for {
+                          ident <- personIdentservice.getPersonIdentOrDefault(committer.name)
+                          _     <- writeXml(categoryFile.toJava, xml, s"Archived technique category: ${catGitPath}")
+                          _     <- IOResult.attempt(gitRepo.git.add.addFilepattern(catGitPath).call())
+                          _     <- IOResult.attempt(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call())
+                        } yield ()
+                      }
+        } yield ()).chainError(s"error when committing technique category '${catGitPath}'").unit
+    }
+  }
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitParseRudderObjects.scala

@@ -42,6 +42,7 @@ import com.normation.GitVersion.Revision
 import com.normation.GitVersion.RevisionInfo
 import com.normation.box.IOManaged
 import com.normation.cfclerk.domain.Technique
+import com.normation.cfclerk.domain.TechniqueCategoryMetadata
 import com.normation.cfclerk.domain.TechniqueCategoryName
 import com.normation.cfclerk.domain.TechniqueId
 import com.normation.cfclerk.domain.TechniqueName
@@ -441,15 +442,24 @@ trait TechniqueRevisionRepository {
    * Directories are added at the beginning
    */
   def getTechniqueFileContents(id: TechniqueId): IOResult[Option[Seq[(String, Option[IOManaged[InputStream]])]]]
+
+  /*
+   * Always use git, does not look at what is on the FS even when revision is default.
+   * Retrieve the category object from the category.xml files under given path.
+   * Path is relative to technique directory root, so that for ex,
+   * `systemSettings/remoteAccess` will look for
+   * `/var/rudder/configuration-repository/techniques/systemSettings/remoteAccess/category.xml`
+   */
+  def getTechniqueCategoryMetadata(path: String, rev: Revision): IOResult[Option[TechniqueCategoryMetadata]]
 }
 
 class GitParseTechniqueLibrary(
-    techniqueParser:  TechniqueParser,
-    val repo:         GitRepositoryProvider,
-    revisionProvider: GitRevisionProvider,
-    libRootDirectory: String, // relative name to git root file
-
-    techniqueMetadata: String
+    techniqueParser:           TechniqueParser,
+    val repo:                  GitRepositoryProvider,
+    revisionProvider:          GitRevisionProvider,
+    libRootDirectory:          String, // relative name to git root file
+    techniqueMetadata:         String,
+    techniqueCategoryFilename: String = "category.xml"
 ) extends TechniqueRevisionRepository {
 
   /**
@@ -502,6 +512,52 @@ class GitParseTechniqueLibrary(
     )
   }
 
+  override def getTechniqueCategoryMetadata(catPath: String, rev: Revision): IOResult[Option[TechniqueCategoryMetadata]] = {
+    val root     = GitRootCategory.getGitDirectoryPath(libRootDirectory).root
+    val filePath = catPath + "/" + techniqueCategoryFilename
+    (for {
+      _      <- ConfigurationLoggerPure.revision.debug(s"Looking for technique category: ${filePath}")
+      treeId <- GitFindUtils.findRevTreeFromRevision(repo.db, rev, revisionProvider.currentRevTreeId)
+      _      <- ConfigurationLoggerPure.revision.trace(s"Git tree corresponding to revision: ${rev.value}: ${treeId.toString}")
+      paths  <- GitFindUtils.listFiles(repo.db, treeId, List(root), List(filePath))
+      _      <- ConfigurationLoggerPure.revision.trace(s"Found candidate paths: ${paths}")
+      data   <- paths.size match {
+                  case 0 =>
+                    ConfigurationLoggerPure.revision.debug(s"Technique category ${filePath} not found") *>
+                    None.succeed
+                  case 1 =>
+                    val gitPath = paths.head
+                    val catId   = catPath.split("/").last
+                    ConfigurationLoggerPure.revision.trace(
+                      s"Technique category ${filePath} found at path '${gitPath}', loading it'"
+                    ) *>
+                    (for {
+                      xml <- GitFindUtils.getFileContent(repo.db, treeId, gitPath) { inputStream =>
+                               ParseXml(inputStream, Some(gitPath)).chainError(s"Error when parsing file '${gitPath}' as XML")
+                             }
+                    } yield {
+                      Some(TechniqueCategoryMetadata.parseXML(xml, catId))
+                    }).tapError(err => {
+                      ConfigurationLoggerPure.revision.debug(
+                        s"Impossible to find technique category with path/revision: '${filePath}/${rev.value}': ${err.fullMsg}."
+                      )
+                    })
+                  case _ =>
+                    Unexpected(
+                      s"There is more than one technique category with path '${filePath}' in git: ${paths.mkString(",")}"
+                    ).fail
+                }
+    } yield {
+      data
+    }).tapBoth(
+      err => ConfigurationLoggerPure.error(err.fullMsg),
+      {
+        case None    => ConfigurationLoggerPure.revision.debug(s" -> not found")
+        case Some(_) => ConfigurationLoggerPure.revision.debug(s" -> found it!")
+      }
+    )
+  }
+
   override def getTechniqueRevision(name: TechniqueName, version: Version): IOResult[List[RevisionInfo]] = {
     val root = GitRootCategory.getGitDirectoryPath(libRootDirectory).root
     for {
@@ -530,6 +586,7 @@ class GitParseTechniqueLibrary(
     } yield {
       revs.toList
     }
+
   }
 
   /*