Skip to content

Commit

Permalink
Work in progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@fanf
fanf committed Jul 4, 2022
1 parent 101e2da commit 6c8ec23
Show file tree
Hide file tree
Showing 6 changed files with 182 additions and 123 deletions.
120 changes: 3 additions & 117 deletions .../sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala
View file
Expand Up @@ -45,7 +45,6 @@ import com.normation.cfclerk.domain.TechniqueName
import com.normation.cfclerk.domain.TechniqueVersion
import com.normation.cfclerk.services.TechniqueRepository
import com.normation.cfclerk.services.UpdateTechniqueLibrary
import com.normation.cfclerk.xmlparsers.TechniqueParser
import com.normation.eventlog.EventActor
import com.normation.eventlog.ModificationId
import com.normation.inventory.domain.AgentType
Expand All @@ -56,18 +55,14 @@ import com.normation.rudder.domain.logger.TimingDebugLoggerPure
import com.normation.rudder.domain.policies.DeleteDirectiveDiff
import com.normation.rudder.domain.policies.Directive
import com.normation.rudder.domain.workflows.ConfigurationChangeRequest
import com.normation.rudder.git.GitConfigItemRepository
import com.normation.rudder.git.GitRepositoryProvider
import com.normation.rudder.hooks.Cmd
import com.normation.rudder.hooks.RunNuCommand
import com.normation.rudder.ncf.ParameterType.ParameterTypeService
import com.normation.rudder.repository.GitModificationRepository
import com.normation.rudder.repository.RoDirectiveRepository
import com.normation.rudder.repository.WoDirectiveRepository
import com.normation.rudder.repository.xml.RudderPrettyPrinter
import com.normation.rudder.repository.xml.XmlArchiverUtils
import com.normation.rudder.repository.xml.TechniqueArchiver
import com.normation.rudder.services.policies.InterpolatedValueCompiler
import com.normation.rudder.services.user.PersonIdentService
import com.normation.rudder.services.workflows.ChangeRequestService
import com.normation.rudder.services.workflows.WorkflowLevelService
import com.normation.utils.Control
Expand All @@ -84,8 +79,6 @@ import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths
import scala.xml.NodeSeq
import scala.xml.Source
import scala.xml.XML
import scala.xml.{Node => XmlNode}

import zio._
Expand Down Expand Up @@ -241,16 +234,7 @@ trait TechniqueWriter {
def writeTechnique(technique: EditorTechnique, methods: Map[BundleName, GenericMethod], modId: ModificationId, committer: EventActor): IOResult[EditorTechnique]
}

/*
* There is a low level API that works at "metadata.xml and files" level and only checks things like metadata.xml ok, overwriting ok,
* unicity, resources presents, agent file presents, etc
*
* This service is also in charge of low-level git related actions: delete files, commit, etc.
*/
trait TechniqueArchiver {
def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit]
def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit]
}


class TechniqueWriterImpl (
archiver : TechniqueArchiver
Expand Down Expand Up @@ -432,7 +416,7 @@ class TechniqueWriterImpl (
id <- TechniqueVersion.parse(technique.version.value).toIO.map(v => TechniqueId(TechniqueName(technique.bundleName.value), v))
// resources files are missing the the "resources/" prefix
resources = technique.ressources.map(r => ResourceFile("resources/" + r.path, r.state))
commit <- archiver.saveTechnique(id, technique.category.split('/'), Chunk.fromIterable(resources), modId, committer, s"Committing technique ${technique.name}")
commit <- archiver.saveTechnique(id, technique.category.split('/').toIndexedSeq, Chunk.fromIterable(resources), modId, committer, s"Committing technique ${technique.name}")
time_4 <- currentTimeMillis
_ <- TimingDebugLoggerPure.trace(s"writeTechnique: committing technique '${technique.name}' took ${time_4 - time_3}ms")
_ <- TimingDebugLoggerPure.debug(s"writeTechnique: writing technique '${technique.name}' took ${time_4 - time_0}ms")
Expand Down Expand Up @@ -1040,102 +1024,4 @@ class DSCTechniqueWriter(

}

/*
* List of files to add/delete in the commit.
* All path are given relative to git root, ie they can be used in
* a git command as they are.
*/
final case class TechniqueFilesToCommit(
add : Chunk[String]
, delete: Chunk[String]
)

class TechniqueArchiverImpl (
override val gitRepo : GitRepositoryProvider
, override val xmlPrettyPrinter : RudderPrettyPrinter
, override val gitModificationRepository : GitModificationRepository
, personIdentservice : PersonIdentService
, techniqueParser : TechniqueParser
, override val groupOwner : String
) extends GitConfigItemRepository with XmlArchiverUtils with TechniqueArchiver {

override val encoding : String = "UTF-8"

// we can't use "techniques" for relative path because of ncf and dsc files. This is an architecture smell, we need to clean it.
override val relativePath = "/"

def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = {
(for {
ident <- personIdentservice.getPersonIdentOrDefault(committer.name)
// construct the path to the technique. Root category is "/", so we filter out all / to be sure
categoryPath <- categories.filter(_ != "/").mkString("/").succeed
rm <- IOResult.effect(gitRepo.git.rm.addFilepattern(s"techniques/${categoryPath}/${techniqueId.serialize}").call())

commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call())
} yield {
s"techniques/${categoryPath}/${techniqueId.serialize}"
}).chainError(s"error when deleting and committing Technique '${techniqueId.serialize}").unit
}

/*
* Return the list of files to commit for the technique.
* For resources, we need to have an hint about the state because we can't guess for deleted files.
*
* TODO: resource files path and status should be deducted from the metadata.xml and the Ressources ID.
* It would insure that there is consistency between technique descriptor and actual content, and
* insure that the lower generation part has a clear and defined API, and that we can do whatever we want
* in the middle.
* All path are related to git repository root path (ie, the correct path to use in git command).
* gitTechniquePath is the path for the technique relative to that git root, without ending slash
*/
def getFilesToCommit(techniqueId: TechniqueId, gitTechniquePath: String, metadata: XmlNode, resourcesStatus: Chunk[ResourceFile]): PureResult[TechniqueFilesToCommit] = {
// parse metadata.xml and find what files need to be added
for {
technique <- techniqueParser.parseXml(metadata, techniqueId)
} yield {
val filesToAdd = (
"metadata.xml" +:
"rudder_reporting.cf" +:
"technique.cf" +:
"technique.ps1" +:
"technique.json" +:
"technique.rd" +:
resourcesStatus.collect {
case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified => path
}
).map(p => gitTechniquePath + "/" + p) // from path relative to technique to relative to git root


// apart resources, additional files to delete are for migration purpose.
val filesToDelete = (
s"ncf/50_techniques/${techniqueId.name.value}" +:
s"dsc/ncf/50_techniques/${techniqueId.name.value}" +:
resourcesStatus.collect {
case ResourceFile(path, ResourceFileState.Deleted) => gitTechniquePath + "/" + path
}
)

TechniqueFilesToCommit(filesToAdd, filesToDelete)
}
}

override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = {

val categoryPath = categories.filter(_ != "/").mkString("/")
val techniqueGitPath = s"techniques/${categoryPath}/${techniqueId.serialize}"

(for {
metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava)))
files <- getFilesToCommit(techniqueId, techniqueGitPath, metadata, resourcesStatus).toIO
ident <- personIdentservice.getPersonIdentOrDefault(committer.name)
added <- ZIO.foreach(files.add) { f =>
IOResult.effect(gitRepo.git.add.addFilepattern(f).call())
}
removed <- ZIO.foreach(files.delete) { f =>
IOResult.effect(gitRepo.git.rm.addFilepattern(f).call())
}
commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call())
} yield ()).chainError(s"error when committing Technique '${techniqueId.serialize}'").unit
}

}
126 changes: 126 additions & 0 deletions .../rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitArchivers.scala
View file
Expand Up @@ -39,10 +39,14 @@ package com.normation.rudder.repository.xml

import com.normation.NamedZioLogger
import com.normation.cfclerk.domain.SectionSpec
import com.normation.cfclerk.domain.Technique
import com.normation.cfclerk.domain.TechniqueId
import com.normation.cfclerk.domain.TechniqueName
import com.normation.cfclerk.services.TechniqueRepository
import com.normation.cfclerk.xmlparsers.TechniqueParser
import com.normation.eventlog.EventActor
import com.normation.eventlog.ModificationId
import com.normation.inventory.domain.AgentType
import com.normation.rudder.domain.Constants.CONFIGURATION_RULES_ARCHIVE_TAG
import com.normation.rudder.domain.Constants.GROUPS_ARCHIVE_TAG
import com.normation.rudder.domain.Constants.PARAMETERS_ARCHIVE_TAG
Expand All @@ -58,15 +62,20 @@ import com.normation.rudder.git.GitArchiverFullCommitUtils
import com.normation.rudder.git.GitConfigItemRepository
import com.normation.rudder.git.GitPath
import com.normation.rudder.git.GitRepositoryProvider
import com.normation.rudder.ncf.ResourceFile
import com.normation.rudder.ncf.ResourceFileState
import com.normation.rudder.repository._
import com.normation.rudder.services.marshalling._
import com.normation.rudder.services.user.PersonIdentService

import net.liftweb.common._
import org.apache.commons.io.FileUtils
import org.eclipse.jgit.lib.PersonIdent

import java.io.File
import scala.collection.mutable.Buffer
import scala.xml.Source
import scala.xml.XML

import zio._
import zio.syntax._
Expand Down Expand Up @@ -169,10 +178,127 @@ trait BuildCategoryPathName[T] {
}


///////////////////////////////////////////////////////////////
////// Archive techniques (techniques, resources files //////
///////////////////////////////////////////////////////////////

/*
* There is a low level API that works at "metadata.xml and files" level and only checks things like metadata.xml ok, overwriting ok,
* unicity, resources presents, agent file presents, etc
*
* This service is also in charge of low-level git related actions: delete files, commit, etc.
*/
trait TechniqueArchiver {
def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit]
def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit]
}

/*
* List of files to add/delete in the commit.
* All path are given relative to git root, ie they can be used in
* a git command as they are.
*/
final case class TechniqueFilesToCommit(
add : Chunk[String]
, delete: Chunk[String]
)

class TechniqueArchiverImpl (
override val gitRepo : GitRepositoryProvider
, override val xmlPrettyPrinter : RudderPrettyPrinter
, override val gitModificationRepository : GitModificationRepository
, personIdentservice : PersonIdentService
, techniqueParser : TechniqueParser
, override val groupOwner : String
) extends GitConfigItemRepository with XmlArchiverUtils with TechniqueArchiver {

override val encoding : String = "UTF-8"

// we can't use "techniques" for relative path because of ncf and dsc files. This is an architecture smell, we need to clean it.
override val relativePath = "techniques"

def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = {
(for {
ident <- personIdentservice.getPersonIdentOrDefault(committer.name)
// construct the path to the technique. Root category is "/", so we filter out all / to be sure
categoryPath <- categories.filter(_ != "/").mkString("/").succeed
rm <- IOResult.effect(gitRepo.git.rm.addFilepattern(s"${relativePath}/${categoryPath}/${techniqueId.serialize}").call())

commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call())
} yield {
s"${relativePath}/${categoryPath}/${techniqueId.serialize}"
}).chainError(s"error when deleting and committing Technique '${techniqueId.serialize}").unit
}

/*
* Return the list of files to commit for the technique.
* For resources, we need to have an hint about the state because we can't guess for deleted files.
*
* TODO: resource files path and status should be deducted from the metadata.xml and the Resources ID.
* It would insure that there is consistency between technique descriptor and actual content, and
* insure that the lower generation part has a clear and defined API, and that we can do whatever we want
* in the middle.
* All path are related to git repository root path (ie, the correct path to use in git command).
* gitTechniquePath is the path for the technique relative to that git root, without ending slash
*/
def getFilesToCommit(technique: Technique, gitTechniquePath: String, resourcesStatus: Chunk[ResourceFile]): TechniqueFilesToCommit = {
// parse metadata.xml and find what files need to be added
val filesToAdd = (
"metadata.xml" +: // standard technique API, used for policy generation pipeline
"technique.json" +: // high-level API between technique editor and rudder. Will evolve to .yml
"technique.rd" +: // deprecated in 7.2. Old rudder-lang input for rudderc, will be replace by yml file
(if(technique.agentConfigs.collectFirst(a => a.agentType == AgentType.CfeCommunity || a.agentType == AgentType.CfeEnterprise).nonEmpty) {
Chunk("rudder_reporting.cf", "technique.cf")
} else Chunk()) +:
(if(technique.agentConfigs.collectFirst(_.agentType == AgentType.Dsc).nonEmpty) {
Chunk("technique.ps1")
} else Chunk()) +:
resourcesStatus.collect {
case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified => path
}
).map(p => gitTechniquePath + "/" + p) // from path relative to technique to relative to git root


// apart resources, additional files to delete are for migration purpose.
val filesToDelete = (
s"ncf/50_techniques/${technique.id.name.value}" +: // added in 5.1 for migration to 6.0
s"dsc/ncf/50_techniques/${technique.id.name.value}" +: // added in 6.1
resourcesStatus.collect {
case ResourceFile(path, ResourceFileState.Deleted) => gitTechniquePath + "/" + path
}
)

TechniqueFilesToCommit(filesToAdd, filesToDelete)
}

override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = {

val categoryPath = categories.filter(_ != "/").mkString("/")
val techniqueGitPath = s"${relativePath}/${categoryPath}/${techniqueId.serialize}"

(for {
metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava)))
tech <- techniqueParser.parseXml(metadata, techniqueId).toIO
files = getFilesToCommit(tech, techniqueGitPath, resourcesStatus)
ident <- personIdentservice.getPersonIdentOrDefault(committer.name)
added <- ZIO.foreach(files.add) { f =>
IOResult.effect(gitRepo.git.add.addFilepattern(f).call())
}
removed <- ZIO.foreach(files.delete) { f =>
IOResult.effect(gitRepo.git.rm.addFilepattern(f).call())
}
commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call())
} yield ()).chainError(s"error when committing Technique '${techniqueId.serialize}'").unit
}

}


///////////////////////////////////////////////////////////////////////////////////////////////
////// Archive the active technique library (categories, techniques, directives) //////
///////////////////////////////////////////////////////////////////////////////////////////////


/**
* A specific trait to create archive of an active technique category.
*
Expand Down
5 changes: 3 additions & 2 deletions ...udder/rudder-core/src/test/scala/com/normation/rudder/ncf/TestEditorTechniqueWriter.scala
View file
Expand Up @@ -90,6 +90,7 @@ import com.normation.rudder.domain.policies.DirectiveId
import com.normation.rudder.domain.policies.DirectiveSaveDiff
import com.normation.rudder.domain.policies.RuleUid
import com.normation.rudder.repository.WoDirectiveRepository
import com.normation.rudder.repository.xml.TechniqueArchiver
import com.normation.rudder.services.nodes.PropertyEngineServiceImpl

import org.specs2.matcher.ContentMatchers
Expand Down Expand Up @@ -123,8 +124,8 @@ class TestEditorTechniqueWriter extends Specification with ContentMatchers with

val expectedPath = "src/test/resources/configuration-repository"
object TestTechniqueArchiver extends TechniqueArchiver {
def saveTechnique(technique: EditorTechnique, modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = UIO.unit
def deleteTechnique(techniqueName: String, techniqueVersion: String, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String): IOResult[Unit] = UIO.unit
override def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String): IOResult[Unit] = UIO.unit
override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String): IOResult[Unit] = UIO.unit
}

object TestLibUpdater extends UpdateTechniqueLibrary {
Expand Down
1 change: 0 additions & 1 deletion ...sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala
View file
Expand Up @@ -39,7 +39,6 @@ package com.normation.rudder.rest.lift

import com.normation.cfclerk.domain.Technique
import com.normation.cfclerk.domain.TechniqueId
import com.normation.cfclerk.services.TechniqueRepository
import com.normation.cfclerk.xmlparsers.TechniqueParser
import com.normation.rudder.api.ApiVersion
import com.normation.rudder.apidata.JsonResponseObjects.JRDirective
Expand Down

0 comments on commit 6c8ec23

Please sign in to comment.