diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala index 8f129b94903..631be0bf6f6 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala @@ -45,7 +45,6 @@ import com.normation.cfclerk.domain.TechniqueName import com.normation.cfclerk.domain.TechniqueVersion import com.normation.cfclerk.services.TechniqueRepository import com.normation.cfclerk.services.UpdateTechniqueLibrary -import com.normation.cfclerk.xmlparsers.TechniqueParser import com.normation.eventlog.EventActor import com.normation.eventlog.ModificationId import com.normation.inventory.domain.AgentType @@ -56,18 +55,14 @@ import com.normation.rudder.domain.logger.TimingDebugLoggerPure import com.normation.rudder.domain.policies.DeleteDirectiveDiff import com.normation.rudder.domain.policies.Directive import com.normation.rudder.domain.workflows.ConfigurationChangeRequest -import com.normation.rudder.git.GitConfigItemRepository -import com.normation.rudder.git.GitRepositoryProvider import com.normation.rudder.hooks.Cmd import com.normation.rudder.hooks.RunNuCommand import com.normation.rudder.ncf.ParameterType.ParameterTypeService -import com.normation.rudder.repository.GitModificationRepository import com.normation.rudder.repository.RoDirectiveRepository import com.normation.rudder.repository.WoDirectiveRepository import com.normation.rudder.repository.xml.RudderPrettyPrinter -import com.normation.rudder.repository.xml.XmlArchiverUtils +import com.normation.rudder.repository.xml.TechniqueArchiver import com.normation.rudder.services.policies.InterpolatedValueCompiler -import com.normation.rudder.services.user.PersonIdentService import com.normation.rudder.services.workflows.ChangeRequestService import com.normation.rudder.services.workflows.WorkflowLevelService import com.normation.utils.Control @@ -84,8 +79,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Paths import scala.xml.NodeSeq -import scala.xml.Source -import scala.xml.XML import scala.xml.{Node => XmlNode} import zio._ @@ -241,16 +234,7 @@ trait TechniqueWriter { def writeTechnique(technique: EditorTechnique, methods: Map[BundleName, GenericMethod], modId: ModificationId, committer: EventActor): IOResult[EditorTechnique] } -/* - * There is a low level API that works at "metadata.xml and files" level and only checks things like metadata.xml ok, overwriting ok, - * unicity, resources presents, agent file presents, etc - * - * This service is also in charge of low-level git related actions: delete files, commit, etc. - */ -trait TechniqueArchiver { - def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] - def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] -} + class TechniqueWriterImpl ( archiver : TechniqueArchiver @@ -432,7 +416,7 @@ class TechniqueWriterImpl ( id <- TechniqueVersion.parse(technique.version.value).toIO.map(v => TechniqueId(TechniqueName(technique.bundleName.value), v)) // resources files are missing the the "resources/" prefix resources = technique.ressources.map(r => ResourceFile("resources/" + r.path, r.state)) - commit <- archiver.saveTechnique(id, technique.category.split('/'), Chunk.fromIterable(resources), modId, committer, s"Committing technique ${technique.name}") + commit <- archiver.saveTechnique(id, technique.category.split('/').toIndexedSeq, Chunk.fromIterable(resources), modId, committer, s"Committing technique ${technique.name}") time_4 <- currentTimeMillis _ <- TimingDebugLoggerPure.trace(s"writeTechnique: committing technique '${technique.name}' took ${time_4 - time_3}ms") _ <- TimingDebugLoggerPure.debug(s"writeTechnique: writing technique '${technique.name}' took ${time_4 - time_0}ms") @@ -1040,102 +1024,4 @@ class DSCTechniqueWriter( } -/* - * List of files to add/delete in the commit. - * All path are given relative to git root, ie they can be used in - * a git command as they are. - */ -final case class TechniqueFilesToCommit( - add : Chunk[String] - , delete: Chunk[String] -) - -class TechniqueArchiverImpl ( - override val gitRepo : GitRepositoryProvider - , override val xmlPrettyPrinter : RudderPrettyPrinter - , override val gitModificationRepository : GitModificationRepository - , personIdentservice : PersonIdentService - , techniqueParser : TechniqueParser - , override val groupOwner : String -) extends GitConfigItemRepository with XmlArchiverUtils with TechniqueArchiver { - - override val encoding : String = "UTF-8" - - // we can't use "techniques" for relative path because of ncf and dsc files. This is an architecture smell, we need to clean it. - override val relativePath = "/" - - def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = { - (for { - ident <- personIdentservice.getPersonIdentOrDefault(committer.name) - // construct the path to the technique. Root category is "/", so we filter out all / to be sure - categoryPath <- categories.filter(_ != "/").mkString("/").succeed - rm <- IOResult.effect(gitRepo.git.rm.addFilepattern(s"techniques/${categoryPath}/${techniqueId.serialize}").call()) - - commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call()) - } yield { - s"techniques/${categoryPath}/${techniqueId.serialize}" - }).chainError(s"error when deleting and committing Technique '${techniqueId.serialize}").unit - } - - /* - * Return the list of files to commit for the technique. - * For resources, we need to have an hint about the state because we can't guess for deleted files. - * - * TODO: resource files path and status should be deducted from the metadata.xml and the Ressources ID. - * It would insure that there is consistency between technique descriptor and actual content, and - * insure that the lower generation part has a clear and defined API, and that we can do whatever we want - * in the middle. - * All path are related to git repository root path (ie, the correct path to use in git command). - * gitTechniquePath is the path for the technique relative to that git root, without ending slash - */ - def getFilesToCommit(techniqueId: TechniqueId, gitTechniquePath: String, metadata: XmlNode, resourcesStatus: Chunk[ResourceFile]): PureResult[TechniqueFilesToCommit] = { - // parse metadata.xml and find what files need to be added - for { - technique <- techniqueParser.parseXml(metadata, techniqueId) - } yield { - val filesToAdd = ( - "metadata.xml" +: - "rudder_reporting.cf" +: - "technique.cf" +: - "technique.ps1" +: - "technique.json" +: - "technique.rd" +: - resourcesStatus.collect { - case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified => path - } - ).map(p => gitTechniquePath + "/" + p) // from path relative to technique to relative to git root - - // apart resources, additional files to delete are for migration purpose. - val filesToDelete = ( - s"ncf/50_techniques/${techniqueId.name.value}" +: - s"dsc/ncf/50_techniques/${techniqueId.name.value}" +: - resourcesStatus.collect { - case ResourceFile(path, ResourceFileState.Deleted) => gitTechniquePath + "/" + path - } - ) - - TechniqueFilesToCommit(filesToAdd, filesToDelete) - } - } - - override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = { - - val categoryPath = categories.filter(_ != "/").mkString("/") - val techniqueGitPath = s"techniques/${categoryPath}/${techniqueId.serialize}" - - (for { - metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava))) - files <- getFilesToCommit(techniqueId, techniqueGitPath, metadata, resourcesStatus).toIO - ident <- personIdentservice.getPersonIdentOrDefault(committer.name) - added <- ZIO.foreach(files.add) { f => - IOResult.effect(gitRepo.git.add.addFilepattern(f).call()) - } - removed <- ZIO.foreach(files.delete) { f => - IOResult.effect(gitRepo.git.rm.addFilepattern(f).call()) - } - commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call()) - } yield ()).chainError(s"error when committing Technique '${techniqueId.serialize}'").unit - } - -} diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitArchivers.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitArchivers.scala index d034644f596..865ba7c5a52 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitArchivers.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitArchivers.scala @@ -39,10 +39,14 @@ package com.normation.rudder.repository.xml import com.normation.NamedZioLogger import com.normation.cfclerk.domain.SectionSpec +import com.normation.cfclerk.domain.Technique import com.normation.cfclerk.domain.TechniqueId import com.normation.cfclerk.domain.TechniqueName import com.normation.cfclerk.services.TechniqueRepository +import com.normation.cfclerk.xmlparsers.TechniqueParser +import com.normation.eventlog.EventActor import com.normation.eventlog.ModificationId +import com.normation.inventory.domain.AgentType import com.normation.rudder.domain.Constants.CONFIGURATION_RULES_ARCHIVE_TAG import com.normation.rudder.domain.Constants.GROUPS_ARCHIVE_TAG import com.normation.rudder.domain.Constants.PARAMETERS_ARCHIVE_TAG @@ -58,8 +62,11 @@ import com.normation.rudder.git.GitArchiverFullCommitUtils import com.normation.rudder.git.GitConfigItemRepository import com.normation.rudder.git.GitPath import com.normation.rudder.git.GitRepositoryProvider +import com.normation.rudder.ncf.ResourceFile +import com.normation.rudder.ncf.ResourceFileState import com.normation.rudder.repository._ import com.normation.rudder.services.marshalling._ +import com.normation.rudder.services.user.PersonIdentService import net.liftweb.common._ import org.apache.commons.io.FileUtils @@ -67,6 +74,8 @@ import org.eclipse.jgit.lib.PersonIdent import java.io.File import scala.collection.mutable.Buffer +import scala.xml.Source +import scala.xml.XML import zio._ import zio.syntax._ @@ -169,10 +178,127 @@ trait BuildCategoryPathName[T] { } +/////////////////////////////////////////////////////////////// +////// Archive techniques (techniques, resources files ////// +/////////////////////////////////////////////////////////////// + +/* + * There is a low level API that works at "metadata.xml and files" level and only checks things like metadata.xml ok, overwriting ok, + * unicity, resources presents, agent file presents, etc + * + * This service is also in charge of low-level git related actions: delete files, commit, etc. + */ +trait TechniqueArchiver { + def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] + def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] +} + +/* + * List of files to add/delete in the commit. + * All path are given relative to git root, ie they can be used in + * a git command as they are. + */ +final case class TechniqueFilesToCommit( + add : Chunk[String] + , delete: Chunk[String] +) + +class TechniqueArchiverImpl ( + override val gitRepo : GitRepositoryProvider + , override val xmlPrettyPrinter : RudderPrettyPrinter + , override val gitModificationRepository : GitModificationRepository + , personIdentservice : PersonIdentService + , techniqueParser : TechniqueParser + , override val groupOwner : String +) extends GitConfigItemRepository with XmlArchiverUtils with TechniqueArchiver { + + override val encoding : String = "UTF-8" + + // we can't use "techniques" for relative path because of ncf and dsc files. This is an architecture smell, we need to clean it. + override val relativePath = "techniques" + + def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = { + (for { + ident <- personIdentservice.getPersonIdentOrDefault(committer.name) + // construct the path to the technique. Root category is "/", so we filter out all / to be sure + categoryPath <- categories.filter(_ != "/").mkString("/").succeed + rm <- IOResult.effect(gitRepo.git.rm.addFilepattern(s"${relativePath}/${categoryPath}/${techniqueId.serialize}").call()) + + commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call()) + } yield { + s"${relativePath}/${categoryPath}/${techniqueId.serialize}" + }).chainError(s"error when deleting and committing Technique '${techniqueId.serialize}").unit + } + + /* + * Return the list of files to commit for the technique. + * For resources, we need to have an hint about the state because we can't guess for deleted files. + * + * TODO: resource files path and status should be deducted from the metadata.xml and the Resources ID. + * It would insure that there is consistency between technique descriptor and actual content, and + * insure that the lower generation part has a clear and defined API, and that we can do whatever we want + * in the middle. + * All path are related to git repository root path (ie, the correct path to use in git command). + * gitTechniquePath is the path for the technique relative to that git root, without ending slash + */ + def getFilesToCommit(technique: Technique, gitTechniquePath: String, resourcesStatus: Chunk[ResourceFile]): TechniqueFilesToCommit = { + // parse metadata.xml and find what files need to be added + val filesToAdd = ( + "metadata.xml" +: // standard technique API, used for policy generation pipeline + "technique.json" +: // high-level API between technique editor and rudder. Will evolve to .yml + "technique.rd" +: // deprecated in 7.2. Old rudder-lang input for rudderc, will be replace by yml file + (if(technique.agentConfigs.collectFirst(a => a.agentType == AgentType.CfeCommunity || a.agentType == AgentType.CfeEnterprise).nonEmpty) { + Chunk("rudder_reporting.cf", "technique.cf") + } else Chunk()) +: + (if(technique.agentConfigs.collectFirst(_.agentType == AgentType.Dsc).nonEmpty) { + Chunk("technique.ps1") + } else Chunk()) +: + resourcesStatus.collect { + case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified => path + } + ).map(p => gitTechniquePath + "/" + p) // from path relative to technique to relative to git root + + + // apart resources, additional files to delete are for migration purpose. + val filesToDelete = ( + s"ncf/50_techniques/${technique.id.name.value}" +: // added in 5.1 for migration to 6.0 + s"dsc/ncf/50_techniques/${technique.id.name.value}" +: // added in 6.1 + resourcesStatus.collect { + case ResourceFile(path, ResourceFileState.Deleted) => gitTechniquePath + "/" + path + } + ) + + TechniqueFilesToCommit(filesToAdd, filesToDelete) + } + + override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = { + + val categoryPath = categories.filter(_ != "/").mkString("/") + val techniqueGitPath = s"${relativePath}/${categoryPath}/${techniqueId.serialize}" + + (for { + metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava))) + tech <- techniqueParser.parseXml(metadata, techniqueId).toIO + files = getFilesToCommit(tech, techniqueGitPath, resourcesStatus) + ident <- personIdentservice.getPersonIdentOrDefault(committer.name) + added <- ZIO.foreach(files.add) { f => + IOResult.effect(gitRepo.git.add.addFilepattern(f).call()) + } + removed <- ZIO.foreach(files.delete) { f => + IOResult.effect(gitRepo.git.rm.addFilepattern(f).call()) + } + commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call()) + } yield ()).chainError(s"error when committing Technique '${techniqueId.serialize}'").unit + } + +} + + /////////////////////////////////////////////////////////////////////////////////////////////// ////// Archive the active technique library (categories, techniques, directives) ////// /////////////////////////////////////////////////////////////////////////////////////////////// + /** * A specific trait to create archive of an active technique category. * diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/ncf/TestEditorTechniqueWriter.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/ncf/TestEditorTechniqueWriter.scala index 99e4ba14ec6..81e7b0a7249 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/ncf/TestEditorTechniqueWriter.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/ncf/TestEditorTechniqueWriter.scala @@ -90,6 +90,7 @@ import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.DirectiveSaveDiff import com.normation.rudder.domain.policies.RuleUid import com.normation.rudder.repository.WoDirectiveRepository +import com.normation.rudder.repository.xml.TechniqueArchiver import com.normation.rudder.services.nodes.PropertyEngineServiceImpl import org.specs2.matcher.ContentMatchers @@ -123,8 +124,8 @@ class TestEditorTechniqueWriter extends Specification with ContentMatchers with val expectedPath = "src/test/resources/configuration-repository" object TestTechniqueArchiver extends TechniqueArchiver { - def saveTechnique(technique: EditorTechnique, modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = UIO.unit - def deleteTechnique(techniqueName: String, techniqueVersion: String, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String): IOResult[Unit] = UIO.unit + override def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String): IOResult[Unit] = UIO.unit + override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String): IOResult[Unit] = UIO.unit } object TestLibUpdater extends UpdateTechniqueLibrary { diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala index 8a668750458..cef1b7dd834 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala @@ -39,7 +39,6 @@ package com.normation.rudder.rest.lift import com.normation.cfclerk.domain.Technique import com.normation.cfclerk.domain.TechniqueId -import com.normation.cfclerk.services.TechniqueRepository import com.normation.cfclerk.xmlparsers.TechniqueParser import com.normation.rudder.api.ApiVersion import com.normation.rudder.apidata.JsonResponseObjects.JRDirective diff --git a/webapp/sources/rudder/rudder-rest/src/test/resources/api/api_groups.yml b/webapp/sources/rudder/rudder-rest/src/test/resources/api/api_groups.yml index 6fdf071a2df..4e4b9c4d1b5 100644 --- a/webapp/sources/rudder/rudder-rest/src/test/resources/api/api_groups.yml +++ b/webapp/sources/rudder/rudder-rest/src/test/resources/api/api_groups.yml @@ -16,7 +16,31 @@ response: "displayName":"Real nodes", "description":"", "category":"GroupRoot", - "nodeIds":[ + "query":{ + "select":"nodeAndPolicyServer", + "composition":"or", + "where":[ + { + "objectType":"node", + "attribute":"nodeId", + "comparator":"eq", + "value":"node1" + }, + { + "objectType":"node", + "attribute":"nodeId", + "comparator":"eq", + "value":"node2" + }, + { + "objectType":"node", + "attribute":"nodeId", + "comparator":"eq", + "value":"root" + } + ] + }, + "nodeIds":[ "node1", "node2", "root" @@ -222,6 +246,30 @@ response: "displayName":"Real nodes", "description":"", "category":"GroupRoot", + "query":{ + "select":"nodeAndPolicyServer", + "composition":"or", + "where":[ + { + "objectType":"node", + "attribute":"nodeId", + "comparator":"eq", + "value":"node1" + }, + { + "objectType":"node", + "attribute":"nodeId", + "comparator":"eq", + "value":"node2" + }, + { + "objectType":"node", + "attribute":"nodeId", + "comparator":"eq", + "value":"root" + } + ] + }, "nodeIds":["node1","node2","root"], "dynamic":false, "enabled":true, diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala index 663b72eeedd..b3c18b4cb62 100644 --- a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala +++ b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala @@ -116,7 +116,6 @@ import com.normation.rudder.ncf import com.normation.rudder.ncf.ParameterType.PlugableParameterTypeService import com.normation.rudder.ncf.ResourceFileService import com.normation.rudder.ncf.RudderCRunner -import com.normation.rudder.ncf.TechniqueArchiverImpl import com.normation.rudder.ncf.TechniqueSerializer import com.normation.rudder.ncf.TechniqueWriter import com.normation.rudder.ncf.TechniqueWriterImpl @@ -1180,7 +1179,7 @@ object RudderConfig extends Loggable { , () => globalComplianceModeService.getGlobalComplianceMode ) - val techniqueArchiver = new TechniqueArchiverImpl(gitConfigRepo, prettyPrinter, gitModificationRepository, personIdentService, RUDDER_GROUP_OWNER_CONFIG_REPO) + val techniqueArchiver = new TechniqueArchiverImpl(gitConfigRepo, prettyPrinter, gitModificationRepository, personIdentService, techniqueParser, RUDDER_GROUP_OWNER_CONFIG_REPO) val techniqueCompiler = new RudderCRunner("/opt/rudder/etc/rudderc.conf","/opt/rudder/bin/rudderc",RUDDER_GIT_ROOT_CONFIG_REPO) val ncfTechniqueWriter: TechniqueWriter = new TechniqueWriterImpl( techniqueArchiver