From 101e2da3ca2f7951d91ce81c50dd36380c40d07d Mon Sep 17 00:00:00 2001 From: "Francois @fanf42 Armand" Date: Mon, 4 Jul 2022 12:17:30 +0200 Subject: [PATCH] wip --- .../rudder/ncf/TechniqueWriter.scala | 215 ++++++++++-------- 1 file changed, 121 insertions(+), 94 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala index 550541f4c80..8f129b94903 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala @@ -38,11 +38,6 @@ package com.normation.rudder.ncf -import better.files.File -import better.files.File.root -import cats.implicits._ - -import com.normation.box._ import com.normation.cfclerk.domain import com.normation.cfclerk.domain.SectionSpec import com.normation.cfclerk.domain.TechniqueId @@ -51,10 +46,6 @@ import com.normation.cfclerk.domain.TechniqueVersion import com.normation.cfclerk.services.TechniqueRepository import com.normation.cfclerk.services.UpdateTechniqueLibrary import com.normation.cfclerk.xmlparsers.TechniqueParser - -import com.normation.errors.IOResult -import com.normation.errors.RudderError -import com.normation.errors._ import com.normation.eventlog.EventActor import com.normation.eventlog.ModificationId import com.normation.inventory.domain.AgentType @@ -81,14 +72,14 @@ import com.normation.rudder.services.workflows.ChangeRequestService import com.normation.rudder.services.workflows.WorkflowLevelService import com.normation.utils.Control -import com.normation.zio.currentTimeMillis +import better.files.File +import better.files.File.root +import cats.implicits._ import net.liftweb.common.Box import net.liftweb.common.EmptyBox import net.liftweb.common.Full import org.joda.time.DateTime -import zio._ -import zio.syntax._ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Paths @@ -97,6 +88,14 @@ import scala.xml.Source import scala.xml.XML import scala.xml.{Node => XmlNode} +import zio._ +import zio.syntax._ +import com.normation.box._ +import com.normation.errors.IOResult +import com.normation.errors.RudderError +import com.normation.errors._ +import com.normation.zio.currentTimeMillis + sealed trait NcfError extends RudderError { def message : String def exception : Option[Throwable] @@ -250,7 +249,7 @@ trait TechniqueWriter { */ trait TechniqueArchiver { def deleteTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] - def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] + def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] } class TechniqueWriterImpl ( @@ -262,7 +261,7 @@ class TechniqueWriterImpl ( , techniqueRepository : TechniqueRepository , workflowLevelService: WorkflowLevelService , xmlPrettyPrinter : RudderPrettyPrinter - , basePath : String + , baseConfigRepoPath : String // root of config repos , parameterTypeService: ParameterTypeService , techniqueSerializer : TechniqueSerializer , compiler : RudderCRunner @@ -275,10 +274,13 @@ class TechniqueWriterImpl ( * or as a fallback on error. * Plus, as of Rudder 7.0, rudderc does not know how to write metadata yet. */ - private[this] val cfengineFallbackTechniqueWriter = new ClassicTechniqueWriter(basePath, parameterTypeService) - private[this] val dscFallbackTechniqueWriter = new DSCTechniqueWriter(basePath, translater, parameterTypeService) + private[this] val cfengineFallbackTechniqueWriter = new ClassicTechniqueWriter(baseConfigRepoPath, parameterTypeService) + private[this] val dscFallbackTechniqueWriter = new DSCTechniqueWriter(baseConfigRepoPath, translater, parameterTypeService) private[this] val agentSpecific = cfengineFallbackTechniqueWriter :: dscFallbackTechniqueWriter :: Nil + // root of technique repository + val techniquesDir = File(baseConfigRepoPath) / "techniques" + override def deleteTechnique(techniqueName : String, techniqueVersion : String, deleteDirective : Boolean, modId : ModificationId, committer : EventActor) : IOResult[Unit] ={ def createCr(directive : Directive, rootSection : SectionSpec ) ={ @@ -305,46 +307,47 @@ class TechniqueWriterImpl ( directives <- readDirectives.getFullDirectiveLibrary().map(_.allActiveTechniques.values.filter(_.techniqueName.value == techniqueId.name.value).flatMap(_.directives).filter(_.techniqueVersion == techniqueId.version)) categories <- techniqueRepository.getTechniqueCategoriesBreadCrump(techniqueId) // Check if we have directives, and either, make an error, if we don't force deletion, or delete them all, creating a change request - _ <- directives match { - case Nil => UIO.unit - case _ => - if (deleteDirective) { - val wf = workflowLevelService.getWorkflowService() - for { - cr <- directives.map(createCr(_,technique.rootSection)).reduceOption(mergeCrs).notOptional(s"Could not create a change request to delete ${technique.name}/${techniqueVersion} directives") - _ <- wf.startWorkflow(cr, committer, Some(s"Deleting technique ${technique.name}/${techniqueVersion}")).toIO - } yield () - } else - Unexpected(s"${directives.size} directives are defined for ${technique.name}/${techniqueVersion} please delete them, or force deletion").fail - } - activeTech <- readDirectives.getActiveTechnique(TechniqueName(technique.name)) + _ <- directives match { + case Nil => UIO.unit + case _ => + if (deleteDirective) { + val wf = workflowLevelService.getWorkflowService() + for { + cr <- directives.map(createCr(_,technique.rootSection)).reduceOption(mergeCrs).notOptional(s"Could not create a change request to delete '${techniqueId.serialize}' directives") + _ <- wf.startWorkflow(cr, committer, Some(s"Deleting technique '${techniqueId.serialize}'")).toIO + } yield () + } else + Unexpected(s"${directives.size} directives are defined for '${techniqueId.serialize}': please delete them, or force deletion").fail + } + activeTech <- readDirectives.getActiveTechnique(techniqueId.name) _ <- activeTech match { - case None => - // No active technique found, let's delete it - ().succeed - case Some(activeTechnique) => - writeDirectives.deleteActiveTechnique(activeTechnique.id, modId, committer, Some(s"Deleting active technique ${techniqueName}")) - } - _ <- archiver.deleteTechnique(technique.name, techniqueVersion, categories.map(_.id.name.value), modId,committer, s"Deleting technique ${technique.name}/${techniqueVersion}") - _ <- techLibUpdate.update(modId, committer, Some(s"Update Technique library after deletion of Technique ${technique.name}")).toIO.chainError( - s"An error occurred during technique update after deletion of Technique ${technique.name}" - ) + case None => + // No active technique found, let's delete it + ().succeed + case Some(activeTechnique) => + writeDirectives.deleteActiveTechnique(activeTechnique.id, modId, committer, Some(s"Deleting active technique '${techniqueId.name.value}'")) + } + _ <- archiver.deleteTechnique(techniqueId, categories.map(_.id.name.value), modId, committer, s"Deleting technique '${techniqueId}'") + _ <- techLibUpdate.update(modId, committer, Some(s"Update Technique library after deletion of technique '${technique.name}'")).toIO.chainError( + s"An error occurred during technique update after deletion of Technique ${technique.name}" + ) } yield () } - def removeInvalidTechnique(basePath: String, techniqueName: String): IOResult[Unit] = { - val unknownTechniquesDir = File(s"${basePath}/techniques/").listRecursively.filter(_.isDirectory).filter(_.name == techniqueName).toList + def removeInvalidTechnique(techniquesDir: File, techniqueId: TechniqueId): IOResult[Unit] = { + val unknownTechniquesDir = techniquesDir.listRecursively.filter(_.isDirectory).filter(_.name == techniqueId.name.value).toList unknownTechniquesDir.length match { case 0 => - ApplicationLogger.info(s"No technique `${techniqueName}` found to delete").succeed + ApplicationLogger.debug(s"No technique `${techniqueId.debugString}` found to delete").succeed case _ => for { _ <- ZIO.foreach(unknownTechniquesDir) { f => - val cat = f.pathAsString.substring(s"${basePath}/techniques/".length).split("/").filter(s => s != techniqueName && s != techniqueVersion).toList + val cat = f.pathAsString.substring((techniquesDir.pathAsString + "/").length).split("/").filter(s => s != techniqueName && s != techniqueVersion).toList for { - _ <- archiver.deleteTechnique(techniqueName, techniqueVersion, cat, modId, committer, s"Deleting invalid technique ${techniqueName}/${techniqueVersion}").chainError( - s"Error when trying to delete invalids techniques, you can manually delete them by running these commands in /var/rudder/configuration-repository/techniques: `rm -rf ${f.pathAsString} && git commit -m 'Deleting invalid technique ${f.pathAsString}' && reload-techniques" - ) + _ <- archiver.deleteTechnique(techniqueId, cat, modId, committer, s"Deleting invalid technique ${techniqueName}/${techniqueVersion}").chainError( + s"Error when trying to delete invalids techniques, you can manually delete them by running these commands in " + + s"${techniquesDir.pathAsString}: `rm -rf ${f.pathAsString} && git commit -m 'Deleting invalid technique ${f.pathAsString}' && reload-techniques" + ) _ <- techLibUpdate.update(modId, committer, Some(s"Update Technique library after deletion of invalid Technique ${techniqueName}")).toIO.chainError( s"An error occurred during technique update after deletion of Technique ${techniqueName}" ) @@ -355,12 +358,12 @@ class TechniqueWriterImpl ( } for { - techVars <- ZIO.fromEither(TechniqueVersion.parse(techniqueVersion)).mapError(Unexpected) - techniqueId = TechniqueId(TechniqueName(techniqueName), techVars) + techVersion <- TechniqueVersion.parse(techniqueVersion).toIO + techniqueId = TechniqueId(TechniqueName(techniqueName), techVersion) _ <- techniqueRepository.get(techniqueId) match { - case Some(technique) => removeTechnique(techniqueId, technique) - case None => removeInvalidTechnique(basePath, techniqueName) - } + case Some(technique) => removeTechnique(techniqueId, technique) + case None => removeInvalidTechnique(techniquesDir, techniqueId) + } } yield () } @@ -426,7 +429,10 @@ class TechniqueWriterImpl ( metadata <- writeMetadata(technique, methods, writtenByRudderWebapp) time_3 <- currentTimeMillis _ <- TimingDebugLoggerPure.trace(s"writeTechnique: generating metadata for technique '${technique.name}' took ${time_3 - time_2}ms") - commit <- archiver.saveTechnique(technique, modId, committer, s"Committing technique ${technique.name}") + id <- TechniqueVersion.parse(technique.version.value).toIO.map(v => TechniqueId(TechniqueName(technique.bundleName.value), v)) + // resources files are missing the the "resources/" prefix + resources = technique.ressources.map(r => ResourceFile("resources/" + r.path, r.state)) + commit <- archiver.saveTechnique(id, technique.category.split('/'), Chunk.fromIterable(resources), modId, committer, s"Committing technique ${technique.name}") time_4 <- currentTimeMillis _ <- TimingDebugLoggerPure.trace(s"writeTechnique: committing technique '${technique.name}' took ${time_4 - time_3}ms") _ <- TimingDebugLoggerPure.debug(s"writeTechnique: writing technique '${technique.name}' took ${time_4 - time_0}ms") @@ -453,7 +459,7 @@ class TechniqueWriterImpl ( val metadataPath = s"techniques/${technique.category}/${technique.bundleName.value}/${technique.version.value}/metadata.xml" - val path = s"${basePath}/${metadataPath}" + val path = s"${baseConfigRepoPath}/${metadataPath}" for { content <- generateTechniqueMetadata(technique, methods, writtenByRudderWebapp).map(n => xmlPrettyPrinter.format(n)).toIO _ <- IOResult.effect(s"An error occurred while creating metadata file for Technique '${technique.name}'") { @@ -567,7 +573,7 @@ class TechniqueWriterImpl ( def writeJson(technique: EditorTechnique, methods: Map[BundleName, GenericMethod]): IOResult[String] = { val metadataPath = s"${technique.path}/technique.json" - val path = s"${basePath}/${metadataPath}" + val path = s"${baseConfigRepoPath}/${metadataPath}" val content = techniqueSerializer.serializeTechniqueMetadata(technique, methods) for { @@ -1034,7 +1040,15 @@ class DSCTechniqueWriter( } - +/* + * List of files to add/delete in the commit. + * All path are given relative to git root, ie they can be used in + * a git command as they are. + */ +final case class TechniqueFilesToCommit( + add : Chunk[String] + , delete: Chunk[String] +) class TechniqueArchiverImpl ( override val gitRepo : GitRepositoryProvider @@ -1063,52 +1077,65 @@ class TechniqueArchiverImpl ( }).chainError(s"error when deleting and committing Technique '${techniqueId.serialize}").unit } - // return file to commit from the technique - def getFilesToCommit(metadata) + /* + * Return the list of files to commit for the technique. + * For resources, we need to have an hint about the state because we can't guess for deleted files. + * + * TODO: resource files path and status should be deducted from the metadata.xml and the Ressources ID. + * It would insure that there is consistency between technique descriptor and actual content, and + * insure that the lower generation part has a clear and defined API, and that we can do whatever we want + * in the middle. + * All path are related to git repository root path (ie, the correct path to use in git command). + * gitTechniquePath is the path for the technique relative to that git root, without ending slash + */ + def getFilesToCommit(techniqueId: TechniqueId, gitTechniquePath: String, metadata: XmlNode, resourcesStatus: Chunk[ResourceFile]): PureResult[TechniqueFilesToCommit] = { + // parse metadata.xml and find what files need to be added + for { + technique <- techniqueParser.parseXml(metadata, techniqueId) + } yield { + val filesToAdd = ( + "metadata.xml" +: + "rudder_reporting.cf" +: + "technique.cf" +: + "technique.ps1" +: + "technique.json" +: + "technique.rd" +: + resourcesStatus.collect { + case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified => path + } + ).map(p => gitTechniquePath + "/" + p) // from path relative to technique to relative to git root - override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = { - val categoryPath = categories.filter(_ != "/").mkString("/") - val techniqueGitPath = s"techniques/${categoryPath}/${techniqueId.serialize}" + // apart resources, additional files to delete are for migration purpose. + val filesToDelete = ( + s"ncf/50_techniques/${techniqueId.name.value}" +: + s"dsc/ncf/50_techniques/${techniqueId.name.value}" +: + resourcesStatus.collect { + case ResourceFile(path, ResourceFileState.Deleted) => gitTechniquePath + "/" + path + } + ) - // parse metadata.xml and find what files need to be added - for { - metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava))) - tech <- techniqueParser.parseXml(metadata, techniqueId) + TechniqueFilesToCommit(filesToAdd, filesToDelete) } + } - val filesToAdd = ( - "metadata.xml" +: - "rudder_reporting.cf" +: - "technique.cf" +: - "technique.ps1" +: - "technique.json" +: - "technique.rd" +: - technique.ressources.collect { - case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified => - s"resources/${path}" - } - ).map(file => s"${techniqueGitPath}/$file") - - // apart resources, additional files to delete are for migration purpose. - val filesToDelete = - s"ncf/50_techniques/${technique.bundleName.value}" +: - s"dsc/ncf/50_techniques/${technique.bundleName.value}" +: - technique.ressources.collect { - case ResourceFile(path, ResourceFileState.Deleted) => - s"${techniqueGitPath}/resources/${path}" - } - (for { - ident <- personIdentservice.getPersonIdentOrDefault(committer.name) + override def saveTechnique(techniqueId: TechniqueId, categories: Seq[String], resourcesStatus: Chunk[ResourceFile], modId: ModificationId, committer: EventActor, msg: String) : IOResult[Unit] = { - added <- ZIO.foreach(filesToAdd) { f => - IOResult.effect(gitRepo.git.add.addFilepattern(f).call()) - } - removed <- ZIO.foreach(filesToDelete) { f => - IOResult.effect(gitRepo.git.rm.addFilepattern(f).call()) - } + val categoryPath = categories.filter(_ != "/").mkString("/") + val techniqueGitPath = s"techniques/${categoryPath}/${techniqueId.serialize}" + + (for { + metadata <- IOResult.effect(XML.load(Source.fromFile((gitRepo.rootDirectory / techniqueGitPath / "metadata.xml").toJava))) + files <- getFilesToCommit(techniqueId, techniqueGitPath, metadata, resourcesStatus).toIO + ident <- personIdentservice.getPersonIdentOrDefault(committer.name) + added <- ZIO.foreach(files.add) { f => + IOResult.effect(gitRepo.git.add.addFilepattern(f).call()) + } + removed <- ZIO.foreach(files.delete) { f => + IOResult.effect(gitRepo.git.rm.addFilepattern(f).call()) + } commit <- IOResult.effect(gitRepo.git.commit.setCommitter(ident).setMessage(msg).call()) - } yield ()).chainError(s"error when committing Technique '${technique.name}/${technique.version}").unit + } yield ()).chainError(s"error when committing Technique '${techniqueId.serialize}'").unit } }