Work in progress

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala

@@ -45,7 +45,7 @@ import com.normation.inventory.domain.AgentType
  * A name, used as an identifier, for a policy.
  * The name must be unique among all policies!
  *
- * TODO : check case sensivity and allowed chars.
+ * TODO : check case sensitivity and allowed chars.
  *
  */
 final case class TechniqueName(value: String) extends AnyVal with Ordered[TechniqueName] {
@@ -63,7 +63,7 @@ final case class TechniqueName(value: String) extends AnyVal with Ordered[Techni
  * among all policies, and a version for that policy.
  */
 final case class TechniqueId(name: TechniqueName, version: TechniqueVersion) extends Ordered[TechniqueId] {
-  // intented for debug/log, not serialization
+  // intended for debug/log, not serialization
   def debugString = serialize
   // a technique
   def serialize = name.value + "/" + version.serialize
@@ -81,6 +81,17 @@ final case class TechniqueId(name: TechniqueName, version: TechniqueVersion) ext
   override def toString: String = serialize
 }
 
+object TechniqueId {
+  def parse(s: String): Either[String, TechniqueId] = {
+    s.split("/").toList match {
+      case n :: v :: Nil =>
+        TechniqueVersion.parse(v).map(x => TechniqueId(TechniqueName(n), x))
+      case _ =>
+        Left(s"Error when parsing '${s}' as a technique id. It should have format 'techniqueName/version+rev' (with +rev optional)")
+    }
+  }
+}
+
 object RunHook {
 /*
  * This data structure holds the agent specific

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala

@@ -67,7 +67,6 @@ import com.normation.zio._
 import zio._
 import zio.syntax._
 import GitTechniqueReader._
-import com.normation.GitVersion
 import com.normation.rudder.domain.logger.TechniqueReaderLoggerPure
 import com.normation.rudder.git.ExactFileTreeFilter
 import com.normation.rudder.git.GitFindUtils
@@ -382,15 +381,12 @@ class GitTechniqueReader(
       }
     }
 
-    //has package id are unique among the whole tree, we are able to find a
-    //template only base on the techniqueId + name.
+    // since package id are unique among the whole tree, we are able to find a
+    // template only base on the techniqueId + name.
 
     val managed = Managed.make(
       for {
-        currentId <- rev match {
-                       case GitVersion.DEFAULT_REV => revisionProvider.currentRevTreeId
-                       case r                      => GitFindUtils.findRevTreeFromRevString(repo.db, r.value)
-                     }
+        currentId <- GitFindUtils.findRevTreeFromRevision(repo.db, rev, revisionProvider.currentRevTreeId)
         optStream <- IOResult.effect {
                        try {
                          //now, the treeWalk

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/git/GitFindUtils.scala

@@ -37,15 +37,18 @@
 
 package com.normation.rudder.git
 
+import com.normation.GitVersion
 import com.normation.GitVersion.Revision
 import com.normation.GitVersion.RevisionInfo
 import com.normation.NamedZioLogger
+
 import com.normation.errors._
 import com.normation.rudder.git.ZipUtils.Zippable
 
 import org.eclipse.jgit.api.Git
 import org.eclipse.jgit.api.Status
 import org.eclipse.jgit.lib.ObjectId
+import org.eclipse.jgit.lib.ObjectStream
 import org.eclipse.jgit.lib.Repository
 import org.eclipse.jgit.lib.{Constants => JConstants}
 import org.eclipse.jgit.revwalk.RevWalk
@@ -57,8 +60,10 @@ import org.joda.time.DateTime
 import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.InputStream
+
 import zio._
 import zio.syntax._
+import com.normation.box.IOManaged
 
 /**
  * Utility trait to find/list/get content
@@ -101,13 +106,17 @@ object GitFindUtils extends NamedZioLogger {
    * relative to git root)
    */
   def getFileContent[T](db:Repository, revTreeId:ObjectId, path:String)(useIt : InputStream => IOResult[T]) : IOResult[T] = {
+    getManagedFileContent(db, revTreeId, path).use(useIt)
+  }
+
+  def getManagedFileContent(db:Repository, revTreeId:ObjectId, path:String): IOManaged[ObjectStream] = {
     val filePath = {
       var p = path
       while (path.endsWith("/")) p = p.substring(0, p.length - 1)
       p
     }
 
-    IOResult.effectM(s"Exception caught when trying to acces file '${filePath}'") {
+    IOManaged.makeM(IOResult.effectM(s"Exception caught when trying to acces file '${filePath}'") {
         //now, the treeWalk
         val tw = new TreeWalk(db)
 
@@ -123,11 +132,11 @@ object GitFindUtils extends NamedZioLogger {
           case Nil =>
             Inconsistency(s"No file were found at path '${filePath}}'").fail
           case h :: Nil =>
-            ZIO.bracket(IOResult.effect(db.open(h).openStream()))(s => effectUioUnit(s.close()))(useIt)
+            IOResult.effect(db.open(h).openStream())
           case _ =>
             Inconsistency(s"More than exactly one matching file were found in the git tree for path '${filePath}', I can not know which one to choose. IDs: ${ids}}").fail
       }
-    }
+    })(s => effectUioUnit(s.close()))
   }
 
   /**
@@ -143,13 +152,25 @@ object GitFindUtils extends NamedZioLogger {
   }
 
   /**
-   * Retrieve the commit tree from a path name.
-   * The path may be any one of `org.eclipse.jgit.lib.Repository#resolve`
+   * Retrieve the revision tree id from a revision.
+   * A default revision must be provided (because default in rudder does not mean the
+   * same as for git)
    */
-  def findRevTreeFromRevString(db:Repository, revString:String) : IOResult[ObjectId] = {
+  def findRevTreeFromRevision(db:Repository, rev: Revision, defaultRev: IOResult[ObjectId]) : IOResult[ObjectId] = {
+    rev match {
+      case GitVersion.DEFAULT_REV => defaultRev
+      case Revision(r) => findRevTreeFromRevString(db, r)
+    }
+  }
+
+  /**
+   * Retrieve the revision tree id from a Git object id
+   */
+  def findRevTreeFromRevString(db:Repository, revString: String) : IOResult[ObjectId] = {
     IOResult.effectM {
       val tree = db.resolve(revString)
       if (null == tree) {
+        Thread.dumpStack()
         Inconsistency(s"The reference branch '${revString}' is not found in the Active Techniques Library's git repository").fail
       } else {
         val rw = new RevWalk(db)
@@ -160,15 +181,26 @@ object GitFindUtils extends NamedZioLogger {
     }
   }
 
+
  /**
   * Get a zip file containing files for commit "revTreeId".
   * You can filter files only some directory by giving
   * a root path.
   */
   def getZip(db:Repository, revTreeId:ObjectId, onlyUnderPaths: List[String] = Nil) : IOResult[Array[Byte]] = {
-    IOResult.effectM(s"Error when creating a zip from files in commit with id: '${revTreeId}'") {
+    for {
+      all      <- getStreamForFiles(db, revTreeId, onlyUnderPaths)
+      zippable =  all.map { case (p, opt) => Zippable(p, opt.map(_.use)) }
+      zip      <- IOResult.effect(new ByteArrayOutputStream()).bracket(os => effectUioUnit(os.close())) { os =>
+                    ZipUtils.zip(os, zippable) *> IOResult.effect(os.toByteArray)
+                  }
+    } yield zip
+  }
+
+  def getStreamForFiles(db:Repository, revTreeId:ObjectId, onlyUnderPaths: List[String] = Nil): IOResult[Seq[(String, Option[IOManaged[InputStream]])]] = {
+    IOResult.effect(s"Error when creating the list of files under ${onlyUnderPaths.mkString(", ")} in commit with id: '${revTreeId}'") {
       val directories = scala.collection.mutable.Set[String]()
-      val zipEntries = scala.collection.mutable.Buffer[Zippable]()
+      val entries = scala.collection.mutable.Buffer.empty[(String, Option[IOManaged[InputStream]])]
       val tw = new TreeWalk(db)
       //create a filter with a OR of all filters
       tw.setFilter(new FileTreeFilter(onlyUnderPaths, Nil))
@@ -178,14 +210,11 @@ object GitFindUtils extends NamedZioLogger {
       while(tw.next) {
         val path = tw.getPathString
         directories += (new File(path)).getParent
-        zipEntries += Zippable(path, Some(GitFindUtils.getFileContent(db,revTreeId,path) _))
+        entries += ((path, Some(GitFindUtils.getManagedFileContent(db,revTreeId,path))))
       }
 
-      //start by creating all directories, then all content
-      val all = directories.map(p => Zippable(p, None)).toSeq ++ zipEntries
-      val out = new ByteArrayOutputStream()
-
-      ZipUtils.zip(out, all) *> IOResult.effect(out.toByteArray())
+      //start by listing all directories, then all content
+      directories.toSeq.map(p => (p, None)) ++ entries
     }
   }
 
@@ -287,4 +316,3 @@ class ExactFileTreeFilter(rootDirectory:Option[String], fileName: String) extend
   override def clone = this
   override lazy val toString = "[.*/%s]".format(fileName)
 }
-

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/xml/GitParseRudderObjects.scala

@@ -57,6 +57,8 @@ import com.normation.rudder.domain.policies.Rule
 import com.normation.rudder.domain.policies.RuleTargetInfo
 import com.normation.rudder.domain.policies.RuleUid
 import com.normation.rudder.domain.properties.GlobalParameter
+import com.normation.rudder.git.ExactFileTreeFilter
+import com.normation.rudder.git.FileTreeFilter
 import com.normation.rudder.git.GitCommitId
 import com.normation.rudder.git.GitFindUtils
 import com.normation.rudder.git.GitRepositoryProvider
@@ -78,11 +80,14 @@ import com.normation.utils.Version
 import com.softwaremill.quicklens._
 import org.eclipse.jgit.lib.ObjectId
 import org.eclipse.jgit.lib.Repository
+import org.eclipse.jgit.treewalk.TreeWalk
 
+import java.io.InputStream
 import java.nio.file.Paths
 
 import zio._
 import zio.syntax._
+import com.normation.box.IOManaged
 import com.normation.errors._
 
 final case class GitRootCategory(
@@ -383,6 +388,15 @@ trait TechniqueRevisionRepository {
    * Get the list of valid revisions for given technique
    */
   def getTechniqueRevision(name: TechniqueName, version: Version): IOResult[List[RevisionInfo]]
+
+  /*
+   * Always use git, does not look at what is on the FS even when revision is default.
+   * Retrieve all files as input streams related to the technique.
+   * Path are relative to technique version directory, so that for ex,
+   * technique/1.0/metadata.xml has path "metadata.xml"
+   * Directories are added at the beginning
+   */
+  def getTechniqueFileContents(id: TechniqueId): IOResult[Option[Seq[(String, Option[IOManaged[InputStream]])]]]
 }
 
 
@@ -397,7 +411,7 @@ class GitParseTechniqueLibrary(
   /**
    * Get a technique for the specific given revision;
    */
-  def getTechnique(name: TechniqueName, version: Version, rev: Revision): IOResult[Option[Technique]] = {
+  override def getTechnique(name: TechniqueName, version: Version, rev: Revision): IOResult[Option[Technique]] = {
     val root = GitRootCategory.getGitDirectoryPath(libRootDirectory).root
     (for {
       v      <- TechniqueVersion(version, rev).left.map(Inconsistency).toIO
@@ -455,9 +469,73 @@ class GitParseTechniqueLibrary(
     } yield {
       revs.toList
     }
+  }
+
+  /*
+   * Always use git, does not look at what is on the FS even when revision is default.
+   * Retrieve all files as input streams related to the technique.
+   * Path are relative to technique version directory, so that for ex,
+   * technique/1.0/metadata.xml has path "metadata.xml"
+   * Directories are added at the beginning
+   */
+  override def getTechniqueFileContents(id: TechniqueId): IOResult[Option[Seq[(String, Option[IOManaged[InputStream]])]]] = {
+    val root = GitRootCategory.getGitDirectoryPath(libRootDirectory).root
+
+    /*
+     * find the path of the technique version
+     */
+    def getFilePath(db: Repository, revTreeId: ObjectId, techniqueId: TechniqueId) = {
+       println(s"***** root: '${root}'; id! ${techniqueId.withDefaultRev.serialize}; treeod: ${revTreeId.toString}")
+      IOResult.effect {
+        //a first walk to find categories
+        val tw = new TreeWalk(db)
+        // there is no directory in git, only files
+        val filter = new FileTreeFilter(List(root + "/"), List(techniqueId.withDefaultRev.serialize + "/" + techniqueMetadata))
+        println(s"**** filter: " + filter)
+        tw.setFilter(filter)
+        tw.setRecursive(true)
+        tw.reset(revTreeId)
+
+        var path = Option.empty[String]
+        while(tw.next && path.isEmpty) {
+          path = Some(tw.getPathString)
+          tw.close()
+        }
+        path.map(_.replaceAll(techniqueMetadata, ""))
+      }
+    }
+
+    for {
+      _       <- ConfigurationLoggerPure.revision.debug(s"Looking for files for technique: ${id.debugString}")
+      treeId  <- GitFindUtils.findRevTreeFromRevision(repo.db, id.version.rev, revisionProvider.currentRevTreeId)
+      _       <- ConfigurationLoggerPure.revision.trace(s"Git tree corresponding to revision: ${id.version.rev.value}: ${treeId.toString}")
+      optPath <- getFilePath(repo.db, treeId, id)
+      all     <- optPath match {
+                   case None       => None.succeed
+                   case Some(path) =>
+                     for {
+                       _       <- ConfigurationLoggerPure.revision.trace(s"Found candidate path for technique ${id.serialize}: ${path}")
+                       all     <- GitFindUtils.getStreamForFiles(repo.db, treeId, List(path))
+                     } yield {
+                       // we need to correct paths to be relative to path
+                       val res = all.flatMap { case (p, opt) =>
+                         val newPath = p.replaceAll("^"+path, "")
+                         newPath.strip() match {
+                           case "" | "/" => None
+                           case x =>
+                             Some((if(x.startsWith("/")) x.tail else x, opt))
+                         }
+                       }
+                       Some(res)
+                     }
+                 }
+    } yield {
+      all
+    }
 
   }
 
+
   def loadTechnique(db: Repository, revTreeId: ObjectId, gitPath: String, id: TechniqueId): IOResult[Technique] = {
     for {
       xml <- GitFindUtils.getFileContent(db, revTreeId, gitPath){ inputStream =>

webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/internal/RulesInternalAPI.scala

@@ -6,16 +6,13 @@ import com.normation.rudder.apidata.JsonResponseObjects.JRRuleNodesDirectives
 import com.normation.rudder.domain.logger.TimingDebugLoggerPure
 import com.normation.rudder.domain.policies.RuleId
 import com.normation.rudder.repository.{RoNodeGroupRepository, RoRuleRepository}
-import com.normation.rudder.rest.{ApiPath, AuthzToken, RestExtractorService, RestUtils, RuleInternalApi => API}
+import com.normation.rudder.rest.{ApiPath, AuthzToken, RestExtractorService, RuleInternalApi => API}
 import com.normation.rudder.rest.lift.{DefaultParams, LiftApiModule, LiftApiModuleProvider, LiftApiModuleString}
 import com.normation.rudder.services.nodes.NodeInfoService
 import com.normation.zio.currentTimeMillis
-import net.liftweb.common.Box
 import net.liftweb.http.{LiftResponse, Req}
 import com.normation.rudder.rest.implicits._
-import net.liftweb.json.JValue
 import com.normation.rudder.apidata.implicits._
-import zio._
 import zio.syntax._
 
 class RulesInternalApi(