diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala index 84cb49f3b50..73c8cb88ad0 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala @@ -45,7 +45,7 @@ import com.normation.inventory.domain.AgentType * A name, used as an identifier, for a policy. * The name must be unique among all policies! * - * TODO : check case sensivity and allowed chars. + * TODO : check case sensitivity and allowed chars. * */ final case class TechniqueName(value: String) extends AnyVal with Ordered[TechniqueName] { @@ -63,7 +63,7 @@ final case class TechniqueName(value: String) extends AnyVal with Ordered[Techni * among all policies, and a version for that policy. */ final case class TechniqueId(name: TechniqueName, version: TechniqueVersion) extends Ordered[TechniqueId] { - // intented for debug/log, not serialization + // intended for debug/log, not serialization def debugString = serialize // a technique def serialize = name.value + "/" + version.serialize @@ -81,6 +81,17 @@ final case class TechniqueId(name: TechniqueName, version: TechniqueVersion) ext override def toString: String = serialize } +object TechniqueId { + def parse(s: String): Either[String, TechniqueId] = { + s.split("/").toList match { + case n :: v :: Nil => + TechniqueVersion.parse(v).map(x => TechniqueId(TechniqueName(n), x)) + case _ => + Left(s"Error when parsing '${s}' as a technique id. It should have format 'techniqueName/version+rev' (with +rev optional)") + } + } +} + object RunHook { /* * This data structure holds the agent specific diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala index b51eaee80e0..f40040f7e41 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala @@ -67,7 +67,6 @@ import com.normation.zio._ import zio._ import zio.syntax._ import GitTechniqueReader._ -import com.normation.GitVersion import com.normation.rudder.domain.logger.TechniqueReaderLoggerPure import com.normation.rudder.git.ExactFileTreeFilter import com.normation.rudder.git.GitFindUtils @@ -382,15 +381,12 @@ class GitTechniqueReader( } } - //has package id are unique among the whole tree, we are able to find a - //template only base on the techniqueId + name. + // since package id are unique among the whole tree, we are able to find a + // template only base on the techniqueId + name. val managed = Managed.make( for { - currentId <- rev match { - case GitVersion.DEFAULT_REV => revisionProvider.currentRevTreeId - case r => GitFindUtils.findRevTreeFromRevString(repo.db, r.value) - } + currentId <- GitFindUtils.findRevTreeFromRevision(repo.db, rev, revisionProvider.currentRevTreeId) optStream <- IOResult.effect { try { //now, the treeWalk diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/git/GitFindUtils.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/git/GitFindUtils.scala index 5c1717b0e38..3690f96b8f9 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/git/GitFindUtils.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/git/GitFindUtils.scala @@ -37,15 +37,18 @@ package com.normation.rudder.git +import com.normation.GitVersion import com.normation.GitVersion.Revision import com.normation.GitVersion.RevisionInfo import com.normation.NamedZioLogger + import com.normation.errors._ import com.normation.rudder.git.ZipUtils.Zippable import org.eclipse.jgit.api.Git import org.eclipse.jgit.api.Status import org.eclipse.jgit.lib.ObjectId +import org.eclipse.jgit.lib.ObjectStream import org.eclipse.jgit.lib.Repository import org.eclipse.jgit.lib.{Constants => JConstants} import org.eclipse.jgit.revwalk.RevWalk @@ -57,8 +60,10 @@ import org.joda.time.DateTime import java.io.ByteArrayOutputStream import java.io.File import java.io.InputStream + import zio._ import zio.syntax._ +import com.normation.box.IOManaged /** * Utility trait to find/list/get content @@ -101,13 +106,17 @@ object GitFindUtils extends NamedZioLogger { * relative to git root) */ def getFileContent[T](db:Repository, revTreeId:ObjectId, path:String)(useIt : InputStream => IOResult[T]) : IOResult[T] = { + getManagedFileContent(db, revTreeId, path).use(useIt) + } + + def getManagedFileContent(db:Repository, revTreeId:ObjectId, path:String): IOManaged[ObjectStream] = { val filePath = { var p = path while (path.endsWith("/")) p = p.substring(0, p.length - 1) p } - IOResult.effectM(s"Exception caught when trying to acces file '${filePath}'") { + IOManaged.makeM(IOResult.effectM(s"Exception caught when trying to acces file '${filePath}'") { //now, the treeWalk val tw = new TreeWalk(db) @@ -123,11 +132,11 @@ object GitFindUtils extends NamedZioLogger { case Nil => Inconsistency(s"No file were found at path '${filePath}}'").fail case h :: Nil => - ZIO.bracket(IOResult.effect(db.open(h).openStream()))(s => effectUioUnit(s.close()))(useIt) + IOResult.effect(db.open(h).openStream()) case _ => Inconsistency(s"More than exactly one matching file were found in the git tree for path '${filePath}', I can not know which one to choose. IDs: ${ids}}").fail } - } + })(s => effectUioUnit(s.close())) } /** @@ -143,13 +152,25 @@ object GitFindUtils extends NamedZioLogger { } /** - * Retrieve the commit tree from a path name. - * The path may be any one of `org.eclipse.jgit.lib.Repository#resolve` + * Retrieve the revision tree id from a revision. + * A default revision must be provided (because default in rudder does not mean the + * same as for git) */ - def findRevTreeFromRevString(db:Repository, revString:String) : IOResult[ObjectId] = { + def findRevTreeFromRevision(db:Repository, rev: Revision, defaultRev: IOResult[ObjectId]) : IOResult[ObjectId] = { + rev match { + case GitVersion.DEFAULT_REV => defaultRev + case Revision(r) => findRevTreeFromRevString(db, r) + } + } + + /** + * Retrieve the revision tree id from a Git object id + */ + def findRevTreeFromRevString(db:Repository, revString: String) : IOResult[ObjectId] = { IOResult.effectM { val tree = db.resolve(revString) if (null == tree) { + Thread.dumpStack() Inconsistency(s"The reference branch '${revString}' is not found in the Active Techniques Library's git repository").fail } else { val rw = new RevWalk(db) @@ -160,15 +181,26 @@ object GitFindUtils extends NamedZioLogger { } } + /** * Get a zip file containing files for commit "revTreeId". * You can filter files only some directory by giving * a root path. */ def getZip(db:Repository, revTreeId:ObjectId, onlyUnderPaths: List[String] = Nil) : IOResult[Array[Byte]] = { - IOResult.effectM(s"Error when creating a zip from files in commit with id: '${revTreeId}'") { + for { + all <- getStreamForFiles(db, revTreeId, onlyUnderPaths) + zippable = all.map { case (p, opt) => Zippable(p, opt.map(_.use)) } + zip <- IOResult.effect(new ByteArrayOutputStream()).bracket(os => effectUioUnit(os.close())) { os => + ZipUtils.zip(os, zippable) *> IOResult.effect(os.toByteArray) + } + } yield zip + } + + def getStreamForFiles(db:Repository, revTreeId:ObjectId, onlyUnderPaths: List[String] = Nil): IOResult[Seq[(String, Option[IOManaged[InputStream]])]] = { + IOResult.effect(s"Error when creating the list of files under ${onlyUnderPaths.mkString(", ")} in commit with id: '${revTreeId}'") { val directories = scala.collection.mutable.Set[String]() - val zipEntries = scala.collection.mutable.Buffer[Zippable]() + val entries = scala.collection.mutable.Buffer.empty[(String, Option[IOManaged[InputStream]])] val tw = new TreeWalk(db) //create a filter with a OR of all filters tw.setFilter(new FileTreeFilter(onlyUnderPaths, Nil)) @@ -178,14 +210,11 @@ object GitFindUtils extends NamedZioLogger { while(tw.next) { val path = tw.getPathString directories += (new File(path)).getParent - zipEntries += Zippable(path, Some(GitFindUtils.getFileContent(db,revTreeId,path) _)) + entries += ((path, Some(GitFindUtils.getManagedFileContent(db,revTreeId,path)))) } - //start by creating all directories, then all content - val all = directories.map(p => Zippable(p, None)).toSeq ++ zipEntries - val out = new ByteArrayOutputStream() - - ZipUtils.zip(out, all) *> IOResult.effect(out.toByteArray()) + //start by listing all directories, then all content + directories.toSeq.map(p => (p, None)) ++ entries } } @@ -287,4 +316,3 @@ class ExactFileTreeFilter(rootDirectory:Option[String], fileName: String) extend override def clone = this override lazy val toString = "[.*/%s]".format(fileName) } - diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitParseRudderObjects.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitParseRudderObjects.scala index dc7c1076698..5f4af91d3d1 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitParseRudderObjects.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitParseRudderObjects.scala @@ -57,6 +57,8 @@ import com.normation.rudder.domain.policies.Rule import com.normation.rudder.domain.policies.RuleTargetInfo import com.normation.rudder.domain.policies.RuleUid import com.normation.rudder.domain.properties.GlobalParameter +import com.normation.rudder.git.ExactFileTreeFilter +import com.normation.rudder.git.FileTreeFilter import com.normation.rudder.git.GitCommitId import com.normation.rudder.git.GitFindUtils import com.normation.rudder.git.GitRepositoryProvider @@ -78,11 +80,14 @@ import com.normation.utils.Version import com.softwaremill.quicklens._ import org.eclipse.jgit.lib.ObjectId import org.eclipse.jgit.lib.Repository +import org.eclipse.jgit.treewalk.TreeWalk +import java.io.InputStream import java.nio.file.Paths import zio._ import zio.syntax._ +import com.normation.box.IOManaged import com.normation.errors._ final case class GitRootCategory( @@ -383,6 +388,15 @@ trait TechniqueRevisionRepository { * Get the list of valid revisions for given technique */ def getTechniqueRevision(name: TechniqueName, version: Version): IOResult[List[RevisionInfo]] + + /* + * Always use git, does not look at what is on the FS even when revision is default. + * Retrieve all files as input streams related to the technique. + * Path are relative to technique version directory, so that for ex, + * technique/1.0/metadata.xml has path "metadata.xml" + * Directories are added at the beginning + */ + def getTechniqueFileContents(id: TechniqueId): IOResult[Option[Seq[(String, Option[IOManaged[InputStream]])]]] } @@ -397,7 +411,7 @@ class GitParseTechniqueLibrary( /** * Get a technique for the specific given revision; */ - def getTechnique(name: TechniqueName, version: Version, rev: Revision): IOResult[Option[Technique]] = { + override def getTechnique(name: TechniqueName, version: Version, rev: Revision): IOResult[Option[Technique]] = { val root = GitRootCategory.getGitDirectoryPath(libRootDirectory).root (for { v <- TechniqueVersion(version, rev).left.map(Inconsistency).toIO @@ -455,9 +469,73 @@ class GitParseTechniqueLibrary( } yield { revs.toList } + } + + /* + * Always use git, does not look at what is on the FS even when revision is default. + * Retrieve all files as input streams related to the technique. + * Path are relative to technique version directory, so that for ex, + * technique/1.0/metadata.xml has path "metadata.xml" + * Directories are added at the beginning + */ + override def getTechniqueFileContents(id: TechniqueId): IOResult[Option[Seq[(String, Option[IOManaged[InputStream]])]]] = { + val root = GitRootCategory.getGitDirectoryPath(libRootDirectory).root + + /* + * find the path of the technique version + */ + def getFilePath(db: Repository, revTreeId: ObjectId, techniqueId: TechniqueId) = { + println(s"***** root: '${root}'; id! ${techniqueId.withDefaultRev.serialize}; treeod: ${revTreeId.toString}") + IOResult.effect { + //a first walk to find categories + val tw = new TreeWalk(db) + // there is no directory in git, only files + val filter = new FileTreeFilter(List(root + "/"), List(techniqueId.withDefaultRev.serialize + "/" + techniqueMetadata)) + println(s"**** filter: " + filter) + tw.setFilter(filter) + tw.setRecursive(true) + tw.reset(revTreeId) + + var path = Option.empty[String] + while(tw.next && path.isEmpty) { + path = Some(tw.getPathString) + tw.close() + } + path.map(_.replaceAll(techniqueMetadata, "")) + } + } + + for { + _ <- ConfigurationLoggerPure.revision.debug(s"Looking for files for technique: ${id.debugString}") + treeId <- GitFindUtils.findRevTreeFromRevision(repo.db, id.version.rev, revisionProvider.currentRevTreeId) + _ <- ConfigurationLoggerPure.revision.trace(s"Git tree corresponding to revision: ${id.version.rev.value}: ${treeId.toString}") + optPath <- getFilePath(repo.db, treeId, id) + all <- optPath match { + case None => None.succeed + case Some(path) => + for { + _ <- ConfigurationLoggerPure.revision.trace(s"Found candidate path for technique ${id.serialize}: ${path}") + all <- GitFindUtils.getStreamForFiles(repo.db, treeId, List(path)) + } yield { + // we need to correct paths to be relative to path + val res = all.flatMap { case (p, opt) => + val newPath = p.replaceAll("^"+path, "") + newPath.strip() match { + case "" | "/" => None + case x => + Some((if(x.startsWith("/")) x.tail else x, opt)) + } + } + Some(res) + } + } + } yield { + all + } } + def loadTechnique(db: Repository, revTreeId: ObjectId, gitPath: String, id: TechniqueId): IOResult[Technique] = { for { xml <- GitFindUtils.getFileContent(db, revTreeId, gitPath){ inputStream => diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/internal/RulesInternalAPI.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/internal/RulesInternalAPI.scala index 04fff743425..f58d5d30614 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/internal/RulesInternalAPI.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/internal/RulesInternalAPI.scala @@ -6,16 +6,13 @@ import com.normation.rudder.apidata.JsonResponseObjects.JRRuleNodesDirectives import com.normation.rudder.domain.logger.TimingDebugLoggerPure import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.repository.{RoNodeGroupRepository, RoRuleRepository} -import com.normation.rudder.rest.{ApiPath, AuthzToken, RestExtractorService, RestUtils, RuleInternalApi => API} +import com.normation.rudder.rest.{ApiPath, AuthzToken, RestExtractorService, RuleInternalApi => API} import com.normation.rudder.rest.lift.{DefaultParams, LiftApiModule, LiftApiModuleProvider, LiftApiModuleString} import com.normation.rudder.services.nodes.NodeInfoService import com.normation.zio.currentTimeMillis -import net.liftweb.common.Box import net.liftweb.http.{LiftResponse, Req} import com.normation.rudder.rest.implicits._ -import net.liftweb.json.JValue import com.normation.rudder.apidata.implicits._ -import zio._ import zio.syntax._ class RulesInternalApi( diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala index c6e1e500415..bc9109b83f9 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ArchiveApi.scala @@ -37,6 +37,8 @@ package com.normation.rudder.rest.lift +import com.normation.cfclerk.domain.Technique +import com.normation.cfclerk.domain.TechniqueId import com.normation.rudder.api.ApiVersion import com.normation.rudder.apidata.JsonResponseObjects.JRRule import com.normation.rudder.apidata.implicits._ @@ -44,9 +46,11 @@ import com.normation.rudder.configuration.ConfigurationRepository import com.normation.rudder.domain.appconfig.FeatureSwitch import com.normation.rudder.domain.logger.ApplicationLogger import com.normation.rudder.domain.logger.ApplicationLoggerPure +import com.normation.rudder.domain.policies.DirectiveId import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.git.ZipUtils import com.normation.rudder.git.ZipUtils.Zippable +import com.normation.rudder.repository.xml.TechniqueRevisionRepository import com.normation.rudder.rest.ApiPath import com.normation.rudder.rest.AuthzToken import com.normation.rudder.rest.RudderJsonResponse @@ -129,17 +133,29 @@ class ArchiveApi( val schema = API.ExportSimple def process0(version: ApiVersion, path: ApiPath, req: Req, params: DefaultParams, authzToken: AuthzToken): LiftResponse = { + def parseRuleIds(req: Req): IOResult[List[RuleId]] = { + ZIO.foreach(req.params.getOrElse("rules", Nil))(RuleId.parse(_).toIO) + } + def parseDirectiveIds(req: Req): IOResult[List[DirectiveId]] = { + ZIO.foreach(req.params.getOrElse("directives", Nil))(DirectiveId.parse(_).toIO) + } + def parseTechniqueIds(req: Req): IOResult[List[TechniqueId]] = { + ZIO.foreach(req.params.getOrElse("techniques", Nil))(TechniqueId.parse(_).toIO) + } +// def parseGroupIds(req: Req): IOResult[List[NodeGroupId]] = { +// ZIO.foreach(req.params.getOrElse("groups", Nil))(NodeGroupId.parse(_).toIO) +// } // lift is not well suited for ZIO... val rootDirName = getArchiveName.runNow - println(s"******* processing request with params = " + req.params) - //do zip val zippables = for { - _ <- ApplicationLoggerPure.Archive.debug(s"Building archive") - ruleIds <- ZIO.foreach(req.params.getOrElse("rules", Nil))(RuleId.parse(_).toIO) - zippables <- archiveBuilderService.buildArchive(rootDirName, ruleIds) + _ <- ApplicationLoggerPure.Archive.debug(s"Building archive") + ruleIds <- parseRuleIds(req) + directiveIds <- parseDirectiveIds(req) + techniquesIds <- parseTechniqueIds(req) + zippables <- archiveBuilderService.buildArchive(rootDirName, techniquesIds, ruleIds) } yield { zippables } @@ -198,7 +214,7 @@ class FileArchiveNameService( ) { def toFileName(name: String): String = { Normalizer.normalize(name, Normalizer.Form.NFKD) - .replaceAll("[^\\p{ASCII}]", "_").take(maxSize) + .replaceAll("""[^\p{Alnum}]""", "_").take(maxSize) } } @@ -218,10 +234,14 @@ class FileArchiveNameService( class ZipArchiveBuilderService( fileArchiveNameService: FileArchiveNameService , configRepo : ConfigurationRepository + , techniqueRevisionRepo : TechniqueRevisionRepository ) { + // names of directories under the root directory of the archive val RULES_DIR = "rules" + val DIRECTIVES_DIR = "directives" + val TECHNIQUES_DIR = "techniques" /* * get the content of the JSON string in the format expected by Zippable @@ -233,31 +253,105 @@ class ZipArchiveBuilderService( } } + + /* + * function that will find the next available name from the given string, + * looking in pool to check for availability (and updating said pool). + * Extension is an extension that should not be normalized. + */ + def findName(origName: String, extension: String, usedNames: Ref[Map[String, Set[String]]], category: String): IOResult[String] = { + def findRecName(s: Set[String], base: String, i: Int): String = { + val n = base+"_"+i + if(s.contains(n)) findRecName(s, base, i+1) else n + } + val name = fileArchiveNameService.toFileName(origName)+extension + + // find a free name, avoiding overwriting a previous similar one + usedNames.modify(m => { + val realName = if( m(category).contains(name) ) { + findRecName(m(category), name, 1) + } else name + ( realName, m + ((category, m(category)+realName)) ) + } ) + } + + /* + * Retrieve the technique using first the cache, then the config service, and update the + * cache accordingly + */ + def getTechnique(techniqueId: TechniqueId, techniques: RefM[Map[TechniqueId, Technique]]): IOResult[Technique] = { + techniques.modify(cache => cache.get(techniqueId) match { + case None => + for { + t <- configRepo.getTechnique(techniqueId).notOptional(s"Technique with id ${techniqueId.serialize} was not found in Rudder") + c = cache + ((t.id, t)) + } yield (t, c) + case Some(t) => + (t, cache).succeed + }) + } + + /* + * Getting technique zippable is more complex than other items because we can have a lot of + * files. The strategy used is to always copy ALL files for the given technique + * TechniquesDir is the path where technqiues are stored, ie for technique "user/1.0", we have: + * techniquesDir / user/1.0/ other techniques file + */ + def getTechniqueZippable(techniquesDir: String, techniqueId: TechniqueId): IOResult[Seq[Zippable]] = { + for { + contents <- techniqueRevisionRepo.getTechniqueFileContents(techniqueId).notOptional(s"Technique with ID '${techniqueId.serialize}' was not found in repository. Please check name and revision.") + } yield { + // we need to change root of zippable + contents.map { case (p, opt) => Zippable(techniquesDir+"/"+p, opt.map(_.use))} + } + } + /* * Prepare the archive. * `rootDirName` is supposed to be normalized, no change will be done with it. * Any missing object will lead to an error. * For each element, an human readable name derived from the object name is used when possible. */ - def buildArchive(rootDirName: String, ruleIds: Seq[RuleId]): IOResult[Chunk[Zippable]] = { + def buildArchive(rootDirName: String, techniqueIds: Seq[TechniqueId], ruleIds: Seq[RuleId]): IOResult[Chunk[Zippable]] = { // normalize to no slash at end val root = rootDirName.strip().replaceAll("""/$""", "") + // rule is easy and independent from other + for { - _ <- ApplicationLoggerPure.Archive.debug(s"Building archive for rules: ${ruleIds.map(_.serialize).mkString(", ")}") - rootZip <- Zippable(rootDirName, None).succeed - rulesDir = root + "/" + RULES_DIR - rulesDirZip = Zippable(rulesDir, None) - rulesZip <- ZIO.foreach(ruleIds) { ruleId => - for { - rule <- configRepo.getRule(ruleId).notOptional(s"Rule with id ${ruleId.serialize} was not found in Rudder") - json = JRRule.fromRule(rule, None, None, None).toJsonPretty - name = fileArchiveNameService.toFileName(rule.name)+".json" - } yield Zippable(name, Some(getJsonZippableContent(json))) - } + // for each kind, we need to keep trace of existing names to avoid overwriting + usedNames <- Ref.make(Map.empty[String, Set[String]]) + _ <- ApplicationLoggerPure.Archive.debug(s"Building archive for rules: ${ruleIds.map(_.serialize).mkString(", ")}") + rootZip <- Zippable(rootDirName, None).succeed + rulesDir = root + "/" + RULES_DIR + _ <- usedNames.update( _ + ((RULES_DIR, Set.empty[String]))) + rulesDirZip = Zippable(rulesDir, None) + rulesZip <- ZIO.foreach(ruleIds) { ruleId => + for { + rule <- configRepo.getRule(ruleId).notOptional(s"Rule with id ${ruleId.serialize} was not found in Rudder") + json = JRRule.fromRule(rule, None, None, None).toJsonPretty + name <- findName(rule.name, ".json", usedNames, RULES_DIR) + } yield Zippable(rulesDir + "/" + name, Some(getJsonZippableContent(json))) + } + + // techniques are a bit complicated: we can have some that will go in the archive, and other + // that we need to retrieve because a directive needs it. So we keep a local cache of retrieved + // techniques. + // Techniques don't need name normalization, their name is already normalized + techniques <- RefM.make(Map.empty[TechniqueId, Technique]) + techniquesDir = root + "/" + TECHNIQUES_DIR + techniquesDirZip = Zippable(techniquesDir, None) + techniquesZip <- ZIO.foreach(techniqueIds) { techniqueId => + for { + technique <- getTechnique(techniqueId, techniques) + techZips <- getTechniqueZippable(techniquesDir, techniqueId) + } yield techZips + } } yield { - Chunk(rootZip, rulesDirZip) ++ rulesZip + Chunk(rootZip, rulesDirZip) ++ rulesZip ++ techniquesZip.flatten } } } + + diff --git a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/ArchiveApiTests.scala b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/ArchiveApiTests.scala index 57c02ad9b7a..108036304fe 100644 --- a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/ArchiveApiTests.scala +++ b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/ArchiveApiTests.scala @@ -118,6 +118,11 @@ class ArchiveApiTests extends Specification with AfterAll with Loggable { "correctly build an archive of one rule" >> { + // rule with ID rule1 defined in com/normation/rudder/MockServices.scala has name: + // 10. Global configuration for all nodes + // so: 10__Global_configuration_for_all_nodes + val fileName = "10__Global_configuration_for_all_nodes.json" + restTest.testGETResponse("/api/archive/export?rules=rule1") { case Full(OutputStreamResponse(out, _, _, _, 200)) => val zipFile = testDir/"archive.zip" @@ -127,8 +132,25 @@ class ArchiveApiTests extends Specification with AfterAll with Loggable { // unzip ZipUtils.unzip(new ZipFile(zipFile.toJava), zipFile.parent.toJava).runNow - (zipFile/"archive").children.toList.map(_.name) must containTheSameElementsAs(List("rule1.json")) - case err => ko(s"I got an error in test: ${err}") + (testDir/"archive/rules").children.toList.map(_.name) must containTheSameElementsAs(List(fileName)) + case err => ko(s"I got an error in test: ${err}") + } + } + + + "correctly build an archive of one technique" >> { + + restTest.testGETResponse("/api/archive/export?techniques=Create_file/1.0") { + case Full(OutputStreamResponse(out, _, _, _, 200)) => + val zipFile = testDir/"archive.zip" + val zipOut = new FileOutputStream(zipFile.toJava) + out(zipOut) + zipOut.close() + // unzip + ZipUtils.unzip(new ZipFile(zipFile.toJava), zipFile.parent.toJava).runNow + + (testDir/"archive/techniques").children.toList.map(_.name) must containTheSameElementsAs(List("Create_file")) + case err => ko(s"I got an error in test: ${err}") } } diff --git a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/RestTestSetUp.scala b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/RestTestSetUp.scala index 7be6439c834..4c42834e3e2 100644 --- a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/RestTestSetUp.scala +++ b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/rest/RestTestSetUp.scala @@ -617,7 +617,7 @@ class RestTestSetUp { val settingsService = new MockSettings(workflowLevelService, new AsyncWorkflowInfo()) object archiveAPIModule { - val archiveBuilderService = new ZipArchiveBuilderService(new FileArchiveNameService(), mockConfigRepo.configurationRepository) + val archiveBuilderService = new ZipArchiveBuilderService(new FileArchiveNameService(), mockConfigRepo.configurationRepository, mockTechniques.techniqueRevisionRepo) val featureSwitchState = Ref.make[FeatureSwitch](FeatureSwitch.Disabled).runNow // fixe archive name to make it simple to test val rootDirName = "archive".succeed @@ -723,6 +723,7 @@ class RestTest(liftRules: LiftRules) { val (p, queryString) = { path.split('?').toList match { + case Nil => (path, "") // should not happen since we have at least path case h :: Nil => (h, "") case h :: tail => (h, tail.mkString("&")) } diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala index 89349e92c7c..c68ea39370e 100644 --- a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala +++ b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala @@ -1326,6 +1326,17 @@ object RudderConfig extends Loggable { ) } + val archiveApi = { + val archiveBuilderService = new ZipArchiveBuilderService(new FileArchiveNameService(), configurationRepository, gitParseTechniqueLibrary) + // fixe archive name to make it simple to test + val rootDirName = "archive".succeed + new com.normation.rudder.rest.lift.ArchiveApi( + archiveBuilderService + , configService.rudder_featureSwitch_archiveApi() + , rootDirName + ) + } + val ApiVersions = ApiVersion(12 , true) :: // rudder 6.0, 6.1 ApiVersion(13 , true) :: // rudder 6.2 @@ -1357,7 +1368,7 @@ object RudderConfig extends Loggable { , new PluginApi(restExtractorService, pluginSettingsService) , new RecentChangesAPI(recentChangesService, restExtractorService) , new RulesInternalApi(restExtractorService, ruleInternalApiService) - , new ArchiveApi(configService.rudder_featureSwitch_archiveApi()) + , archiveApi // info api must be resolved latter, because else it misses plugin apis ! ) diff --git a/webapp/sources/utils/src/main/scala/com/normation/ObjectVersionCommons.scala b/webapp/sources/utils/src/main/scala/com/normation/ObjectVersionCommons.scala index 2487c012c91..8cebe32b112 100644 --- a/webapp/sources/utils/src/main/scala/com/normation/ObjectVersionCommons.scala +++ b/webapp/sources/utils/src/main/scala/com/normation/ObjectVersionCommons.scala @@ -100,7 +100,7 @@ final object GitVersion { case id :: Nil => Right((id, GitVersion.DEFAULT_REV)) case id :: "" :: Nil => Right((id, GitVersion.DEFAULT_REV)) case id :: rev :: Nil => Right((id, Revision(rev))) - case _ => Left(s"Error when parsing '${s}' as a directive id. At most one '+' is authorized.") + case _ => Left(s"Error when parsing '${s}' as a rudder id. At most one '+' is authorized.") } }