Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@fanf
fanf committed Jul 4, 2022
1 parent 0051307 commit 101e2da
Showing 1 changed file with 121 additions and 94 deletions.
215 changes: 121 additions & 94 deletions .../sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@
package com.normation.rudder.ncf


import better.files.File
import better.files.File.root
import cats.implicits._

import com.normation.box._
import com.normation.cfclerk.domain
import com.normation.cfclerk.domain.SectionSpec
import com.normation.cfclerk.domain.TechniqueId
Expand All @@ -51,10 +46,6 @@ import com.normation.cfclerk.domain.TechniqueVersion
import com.normation.cfclerk.services.TechniqueRepository
import com.normation.cfclerk.services.UpdateTechniqueLibrary
import com.normation.cfclerk.xmlparsers.TechniqueParser

import com.normation.errors.IOResult
import com.normation.errors.RudderError
import com.normation.errors._
import com.normation.eventlog.EventActor
import com.normation.eventlog.ModificationId
import com.normation.inventory.domain.AgentType
Expand All @@ -81,14 +72,14 @@ import com.normation.rudder.services.workflows.ChangeRequestService
import com.normation.rudder.services.workflows.WorkflowLevelService
import com.normation.utils.Control

import com.normation.zio.currentTimeMillis
import better.files.File
import better.files.File.root
import cats.implicits._
import net.liftweb.common.Box
import net.liftweb.common.EmptyBox
import net.liftweb.common.Full
import org.joda.time.DateTime

import zio._
import zio.syntax._
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths
Expand All @@ -97,6 +88,14 @@ import scala.xml.Source
import scala.xml.XML
import scala.xml.{Node => XmlNode}

import zio._
import zio.syntax._
import com.normation.box._
import com.normation.errors.IOResult
import com.normation.errors.RudderError
import com.normation.errors._
import com.normation.zio.currentTimeMillis

sealed trait NcfError extends RudderError {
def message : String
def exception : Option[Throwable]
Expand Down Expand Up @@ -250,7 +249,7 @@ trait TechniqueWriter {
*/
trait TechniqueArchiver {
def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit]
def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit]
def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit]
}

class TechniqueWriterImpl (
Expand All @@ -262,7 +261,7 @@ class TechniqueWriterImpl (
, techniqueRepository : TechniqueRepository
, workflowLevelService: WorkflowLevelService
, xmlPrettyPrinter : RudderPrettyPrinter
, basePath : String
, baseConfigRepoPath : String // root of config repos
, parameterTypeService: ParameterTypeService
, techniqueSerializer : TechniqueSerializer
, compiler : RudderCRunner
Expand All @@ -275,10 +274,13 @@ class TechniqueWriterImpl (
* or as a fallback on error.
* Plus, as of Rudder 7.0, rudderc does not know how to write metadata yet.
*/
private[this] val cfengineFallbackTechniqueWriter = new ClassicTechniqueWriter(basePath, parameterTypeService)
private[this] val dscFallbackTechniqueWriter = new DSCTechniqueWriter(basePath, translater, parameterTypeService)
private[this] val cfengineFallbackTechniqueWriter = new ClassicTechniqueWriter(baseConfigRepoPath, parameterTypeService)
private[this] val dscFallbackTechniqueWriter = new DSCTechniqueWriter(baseConfigRepoPath, translater, parameterTypeService)
private[this] val agentSpecific = cfengineFallbackTechniqueWriter :: dscFallbackTechniqueWriter :: Nil

// root of technique repository
val techniquesDir = File(baseConfigRepoPath) / "techniques"

override def deleteTechnique(techniqueName : String, techniqueVersion : String, deleteDirective : Boolean, modId : ModificationId, committer : EventActor) : IOResult[Unit] ={

def createCr(directive : Directive, rootSection : SectionSpec ) ={
Expand All @@ -305,46 +307,47 @@ class TechniqueWriterImpl (
directives <- readDirectives.getFullDirectiveLibrary().map(_.allActiveTechniques.values.filter(_.techniqueName.value == techniqueId.name.value).flatMap(_.directives).filter(_.techniqueVersion == techniqueId.version))
categories <- techniqueRepository.getTechniqueCategoriesBreadCrump(techniqueId)
// Check if we have directives, and either, make an error, if we don't force deletion, or delete them all, creating a change request
_ <- directives match {
case Nil => UIO.unit
case _ =>
if (deleteDirective) {
val wf = workflowLevelService.getWorkflowService()
for {
cr <- directives.map(createCr(_,technique.rootSection)).reduceOption(mergeCrs).notOptional(s"Could not create a change request to delete ${technique.name}/${techniqueVersion} directives")
_ <- wf.startWorkflow(cr, committer, Some(s"Deleting technique ${technique.name}/${techniqueVersion}")).toIO
} yield ()
} else
Unexpected(s"${directives.size} directives are defined for ${technique.name}/${techniqueVersion} please delete them, or force deletion").fail
}
activeTech <- readDirectives.getActiveTechnique(TechniqueName(technique.name))
_ <- directives match {
case Nil => UIO.unit
case _ =>
if (deleteDirective) {
val wf = workflowLevelService.getWorkflowService()
for {
cr <- directives.map(createCr(_,technique.rootSection)).reduceOption(mergeCrs).notOptional(s"Could not create a change request to delete '${techniqueId.serialize}' directives")
_ <- wf.startWorkflow(cr, committer, Some(s"Deleting technique '${techniqueId.serialize}'")).toIO
} yield ()
} else
Unexpected(s"${directives.size} directives are defined for '${techniqueId.serialize}': please delete them, or force deletion").fail
}
activeTech <- readDirectives.getActiveTechnique(techniqueId.name)
_ <- activeTech match {
case None =>
// No active technique found, let's delete it
().succeed
case Some(activeTechnique) =>
writeDirectives.deleteActiveTechnique(activeTechnique.id, modId, committer, Some(s"Deleting active technique ${techniqueName}"))
}
_ <- archiver.deleteTechnique(technique.name, techniqueVersion, categories.map(_.id.name.value), modId,committer, s"Deleting technique ${technique.name}/${techniqueVersion}")
_ <- techLibUpdate.update(modId, committer, Some(s"Update Technique library after deletion of Technique ${technique.name}")).toIO.chainError(
s"An error occurred during technique update after deletion of Technique ${technique.name}"
)
case None =>
// No active technique found, let's delete it
().succeed
case Some(activeTechnique) =>
writeDirectives.deleteActiveTechnique(activeTechnique.id, modId, committer, Some(s"Deleting active technique '${techniqueId.name.value}'"))
}
_ <- archiver.deleteTechnique(techniqueId, categories.map(_.id.name.value), modId, committer, s"Deleting technique '${techniqueId}'")
_ <- techLibUpdate.update(modId, committer, Some(s"Update Technique library after deletion of technique '${technique.name}'")).toIO.chainError(
s"An error occurred during technique update after deletion of Technique ${technique.name}"
)
} yield ()
}

def removeInvalidTechnique(basePath: String, techniqueName: String): IOResult[Unit] = {
val unknownTechniquesDir = File(s"${basePath}/techniques/").listRecursively.filter(_.isDirectory).filter(_.name == techniqueName).toList
def removeInvalidTechnique(techniquesDir: File, techniqueId: TechniqueId): IOResult[Unit] = {
val unknownTechniquesDir = techniquesDir.listRecursively.filter(_.isDirectory).filter(_.name == techniqueId.name.value).toList
unknownTechniquesDir.length match {
case 0 =>
ApplicationLogger.info(s"No technique `${techniqueName}` found to delete").succeed
ApplicationLogger.debug(s"No technique `${techniqueId.debugString}` found to delete").succeed
case _ =>
for {
_ <- ZIO.foreach(unknownTechniquesDir) { f =>
val cat = f.pathAsString.substring(s"${basePath}/techniques/".length).split("/").filter(s => s != techniqueName && s != techniqueVersion).toList
val cat = f.pathAsString.substring((techniquesDir.pathAsString + "/").length).split("/").filter(s => s != techniqueName && s != techniqueVersion).toList
for {
_ <- archiver.deleteTechnique(techniqueName, techniqueVersion, cat, modId, committer, s"Deleting invalid technique ${techniqueName}/${techniqueVersion}").chainError(
s"Error when trying to delete invalids techniques, you can manually delete them by running these commands in /var/rudder/configuration-repository/techniques: `rm -rf ${f.pathAsString} && git commit -m 'Deleting invalid technique ${f.pathAsString}' && reload-techniques"
)
_ <- archiver.deleteTechnique(techniqueId, cat, modId, committer, s"Deleting invalid technique ${techniqueName}/${techniqueVersion}").chainError(
s"Error when trying to delete invalids techniques, you can manually delete them by running these commands in " +
s"${techniquesDir.pathAsString}: `rm -rf ${f.pathAsString} && git commit -m 'Deleting invalid technique ${f.pathAsString}' && reload-techniques"
)
_ <- techLibUpdate.update(modId, committer, Some(s"Update Technique library after deletion of invalid Technique ${techniqueName}")).toIO.chainError(
s"An error occurred during technique update after deletion of Technique ${techniqueName}"
)
Expand All @@ -355,12 +358,12 @@ class TechniqueWriterImpl (
}

for {
techVars <- ZIO.fromEither(TechniqueVersion.parse(techniqueVersion)).mapError(Unexpected)
techniqueId = TechniqueId(TechniqueName(techniqueName), techVars)
techVersion <- TechniqueVersion.parse(techniqueVersion).toIO
techniqueId = TechniqueId(TechniqueName(techniqueName), techVersion)
_ <- techniqueRepository.get(techniqueId) match {
case Some(technique) => removeTechnique(techniqueId, technique)
case None => removeInvalidTechnique(basePath, techniqueName)
}
case Some(technique) => removeTechnique(techniqueId, technique)
case None => removeInvalidTechnique(techniquesDir, techniqueId)
}
} yield ()
}

Expand Down Expand Up @@ -426,7 +429,10 @@ class TechniqueWriterImpl (
metadata <- writeMetadata(technique, methods, writtenByRudderWebapp)
time_3 <- currentTimeMillis
_ <- TimingDebugLoggerPure.trace(s"writeTechnique: generating metadata for technique '${technique.name}' took ${time_3 - time_2}ms")
commit <- archiver.saveTechnique(technique, modId, committer, s"Committing technique ${technique.name}")
id <- TechniqueVersion.parse(technique.version.value).toIO.map(v => TechniqueId(TechniqueName(technique.bundleName.value), v))
// resources files are missing the the "resources/" prefix
resources = technique.ressources.map(r => ResourceFile("resources/" + r.path, r.state))
commit <- archiver.saveTechnique(id, technique.category.split('/'), Chunk.fromIterable(resources), modId, committer, s"Committing technique ${technique.name}")
time_4 <- currentTimeMillis
_ <- TimingDebugLoggerPure.trace(s"writeTechnique: committing technique '${technique.name}' took ${time_4 - time_3}ms")
_ <- TimingDebugLoggerPure.debug(s"writeTechnique: writing technique '${technique.name}' took ${time_4 - time_0}ms")
Expand All @@ -453,7 +459,7 @@ class TechniqueWriterImpl (

val metadataPath = s"techniques/${technique.category}/${technique.bundleName.value}/${technique.version.value}/metadata.xml"

val path = s"${basePath}/${metadataPath}"
val path = s"${baseConfigRepoPath}/${metadataPath}"
for {
content <- generateTechniqueMetadata(technique, methods, writtenByRudderWebapp).map(n => xmlPrettyPrinter.format(n)).toIO
_ <- IOResult.effect(s"An error occurred while creating metadata file for Technique '${technique.name}'") {
Expand Down Expand Up @@ -567,7 +573,7 @@ class TechniqueWriterImpl (
def writeJson(technique: EditorTechnique, methods: Map[BundleName, GenericMethod]): IOResult[String] = {
val metadataPath = s"${technique.path}/technique.json"

val path = s"${basePath}/${metadataPath}"
val path = s"${baseConfigRepoPath}/${metadataPath}"

val content = techniqueSerializer.serializeTechniqueMetadata(technique, methods)
for {
Expand Down Expand Up @@ -1034,7 +1040,15 @@ class DSCTechniqueWriter(

}


/*
* List of files to add/delete in the commit.
* All path are given relative to git root, ie they can be used in
* a git command as they are.
*/
final case class TechniqueFilesToCommit(
add : Chunk[String]
, delete: Chunk[String]
)

class TechniqueArchiverImpl (
override val gitRepo : GitRepositoryProvider
Expand Down Expand Up @@ -1063,52 +1077,65 @@ class TechniqueArchiverImpl (
}).chainError(s"error when deleting and committing Technique '${techniqueId.serialize}").unit
}

// return file to commit from the technique
def getFilesToCommit(metadata)
/*
* Return the list of files to commit for the technique.
* For resources, we need to have an hint about the state because we can't guess for deleted files.
*
* TODO: resource files path and status should be deducted from the metadata.xml and the Ressources ID.
* It would insure that there is consistency between technique descriptor and actual content, and
* insure that the lower generation part has a clear and defined API, and that we can do whatever we want
* in the middle.
* All path are related to git repository root path (ie, the correct path to use in git command).
* gitTechniquePath is the path for the technique relative to that git root, without ending slash
*/
def getFilesToCommit(techniqueId: TechniqueId, gitTechniquePath: String, metadata: XmlNode, resourcesStatus: Chunk[ResourceFile]): PureResult[TechniqueFilesToCommit] = {
// parse metadata.xml and find what files need to be added
for {
technique <- techniqueParser.parseXml(metadata, techniqueId)
} yield {
val filesToAdd = (
"metadata.xml" +:
"rudder_reporting.cf" +:
"technique.cf" +:
"technique.ps1" +:
"technique.json" +:
"technique.rd" +:
resourcesStatus.collect {
case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified => path
}
).map(p => gitTechniquePath + "/" + p) // from path relative to technique to relative to git root

override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = {

val categoryPath = categories.filter(_ != "/").mkString("/")
val techniqueGitPath = s"techniques/${categoryPath}/${techniqueId.serialize}"
// apart resources, additional files to delete are for migration purpose.
val filesToDelete = (
s"ncf/50_techniques/${techniqueId.name.value}" +:
s"dsc/ncf/50_techniques/${techniqueId.name.value}" +:
resourcesStatus.collect {
case ResourceFile(path, ResourceFileState.Deleted) => gitTechniquePath + "/" + path
}
)

// parse metadata.xml and find what files need to be added
for {
metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava)))
tech <- techniqueParser.parseXml(metadata, techniqueId)
TechniqueFilesToCommit(filesToAdd, filesToDelete)
}
}

val filesToAdd = (
"metadata.xml" +:
"rudder_reporting.cf" +:
"technique.cf" +:
"technique.ps1" +:
"technique.json" +:
"technique.rd" +:
technique.ressources.collect {
case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified =>
s"resources/${path}"
}
).map(file => s"${techniqueGitPath}/$file")

// apart resources, additional files to delete are for migration purpose.
val filesToDelete =
s"ncf/50_techniques/${technique.bundleName.value}" +:
s"dsc/ncf/50_techniques/${technique.bundleName.value}" +:
technique.ressources.collect {
case ResourceFile(path, ResourceFileState.Deleted) =>
s"${techniqueGitPath}/resources/${path}"
}
(for {
ident <- personIdentservice.getPersonIdentOrDefault(committer.name)
override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = {

added <- ZIO.foreach(filesToAdd) { f =>
IOResult.effect(gitRepo.git.add.addFilepattern(f).call())
}
removed <- ZIO.foreach(filesToDelete) { f =>
IOResult.effect(gitRepo.git.rm.addFilepattern(f).call())
}
val categoryPath = categories.filter(_ != "/").mkString("/")
val techniqueGitPath = s"techniques/${categoryPath}/${techniqueId.serialize}"

(for {
metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava)))
files <- getFilesToCommit(techniqueId, techniqueGitPath, metadata, resourcesStatus).toIO
ident <- personIdentservice.getPersonIdentOrDefault(committer.name)
added <- ZIO.foreach(files.add) { f =>
IOResult.effect(gitRepo.git.add.addFilepattern(f).call())
}
removed <- ZIO.foreach(files.delete) { f =>
IOResult.effect(gitRepo.git.rm.addFilepattern(f).call())
}
commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call())
} yield ()).chainError(s"error when committing Technique '${technique.name}/${technique.version}").unit
} yield ()).chainError(s"error when committing Technique '${techniqueId.serialize}'").unit
}

}

0 comments on commit 101e2da

Please sign in to comment.