Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixes #19061: Use rudderc to compile technique from the editor instead of rudder logic #3553

Merged
merged 3 commits into from
Jun 11, 2021
Merged

Fixes #19061: Use rudderc to compile technique from the editor instead of rudder logic #3553

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
266b849
Fixes #19061: Use rudderc to compile technique from the editor instea…
VinceMacBuche Mar 23, 2021
8a32de9
fixup! Fixes #19061: Use rudderc to compile technique from the editor…
VinceMacBuche Jun 3, 2021
1904baf
fixup! fixup! Fixes #19061: Use rudderc to compile technique from the…
VinceMacBuche Jun 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions .../com/normation/rudder/ncf/Technique.scala → ...ormation/rudder/ncf/EditorTechnique.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ case class ResourceFile(
, state : ResourceFileState
)

final case class Technique(
final case class EditorTechnique(
bundleName : BundleName
, name : String
, category : String
Expand All @@ -104,7 +104,9 @@ final case class Technique(
, description : String
, parameters : Seq[TechniqueParameter]
, ressources : Seq[ResourceFile]
)
) {
val path = s"techniques/${category}/${bundleName.value}/${version.value}"
}

final case class MethodCall(
methodId : BundleName
Expand Down Expand Up @@ -311,7 +313,7 @@ class TechniqueSerializer(parameterTypeService: ParameterTypeService) {

import net.liftweb.json.JsonDSL._

def serializeTechniqueMetadata(technique: ncf.Technique): JValue = {
def serializeTechniqueMetadata(technique: ncf.EditorTechnique): JValue = {

def serializeTechniqueParameter(parameter: TechniqueParameter): JValue = {
( ("id" -> parameter.id.value)
Expand Down Expand Up @@ -406,7 +408,7 @@ class TechniqueSerializer(parameterTypeService: ParameterTypeService) {
}

class ResourceFileService( gitReposProvider : GitRepositoryProvider) {
def getResources(technique: Technique) = {
def getResources(technique: EditorTechnique) = {
getResourcesFromDir(s"techniques/${technique.category}/${technique.bundleName.value}/${technique.version.value}/resources", technique.bundleName.value, technique.version.value)

}
Expand Down
126 changes: 82 additions & 44 deletions .../sources/rudder/rudder-core/src/main/scala/com/normation/rudder/ncf/TechniqueWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,17 +42,18 @@ import cats.implicits._
import com.normation.errors._
import com.normation.eventlog.EventActor
import com.normation.eventlog.ModificationId
import java.nio.charset.StandardCharsets

import java.nio.charset.StandardCharsets
import com.normation.inventory.domain.AgentType
import com.normation.rudder.repository.GitModificationRepository
import com.normation.rudder.repository.xml.GitArchiverUtils
import com.normation.cfclerk.services.GitRepositoryProvider

import java.io.{File => JFile}
import java.nio.file.Files
import java.nio.file.Paths

import better.files.File
import better.files.File.root
import com.normation.cfclerk.domain.SectionSpec
import com.normation.cfclerk.domain.TechniqueId
import com.normation.cfclerk.domain.TechniqueName
Expand All @@ -79,15 +80,12 @@ import com.normation.rudder.services.workflows.WorkflowLevelService
import com.normation.utils.Control
import net.liftweb.common.Box
import net.liftweb.common.EmptyBox
import com.normation.rudder.domain.logger.ApplicationLogger
import com.normation.rudder.hooks.Cmd
import com.normation.rudder.hooks.RunNuCommand
import com.normation.errors.RudderError
import com.normation.rudder.domain.logger.ApplicationLoggerPure
import com.normation.rudder.repository.WoDirectiveRepository

import scala.jdk.CollectionConverters._
import com.normation.zio._
import org.joda.time.DateTime

trait NcfError extends RudderError {
def message : String
Expand All @@ -99,6 +97,40 @@ final case class IOError(message : String, exception : Option[Throwable]) extend
final case class TechniqueUpdateError(message : String, exception : Option[Throwable]) extends NcfError
final case class MethodNotFound(message : String, exception : Option[Throwable]) extends NcfError

class RudderCRunner (
configFilePath : String
, rudderCPath : String
, outputPath : String
) {
def compileTechnique(technique : EditorTechnique) = {
for {
r <- RunNuCommand.run(Cmd(rudderCPath, "save" :: "-j" :: "-i" :: s""""${outputPath}/${technique.path}/technique.json"""" :: s"--config-file=${configFilePath}" :: Nil, Map.empty))
res <- r.await
_ <- ZIO.when(res.code != 0) {
Inconsistency(
s"An error occurred when translating technique.json file into Rudder language\n code: ${res.code}\n stderr: ${res.stderr}\n stdout: ${res.stdout}"
).fail
}
r <- RunNuCommand.run(Cmd(rudderCPath, "compile" :: "-j" :: "-f" :: """"cf"""" :: "-i" :: s""""${outputPath}/${technique.path}/technique.rl"""" :: s"--config-file=${configFilePath}" :: Nil, Map.empty))
res <- r.await
_ <- ZIO.when(res.code != 0) {
Inconsistency(
s"An error occurred when compiling technique.rl file into cfengine\n code: ${res.code}\n stderr: ${res.stderr}\n stdout: ${res.stdout}"
).fail
}
r <- RunNuCommand.run(Cmd(rudderCPath, "compile" :: "-j" :: "-f" :: """"dsc"""" :: "-i" :: s""""${outputPath}/${technique.path}/technique.rl"""" :: s"--config-file=${configFilePath}" :: Nil, Map.empty))
res <- r.await
_ <- ZIO.when(res.code != 0) {
Inconsistency(
s"An error occurred when compiling technique.rl file into dsc\n code: ${res.code}\n stderr: ${res.stderr}\n stdout: ${res.stdout}"
).fail
}
} yield {
()
}
}
}

class TechniqueWriter (
archiver : TechniqueArchiver
, techLibUpdate : UpdateTechniqueLibrary
Expand All @@ -111,7 +143,8 @@ class TechniqueWriter (
, basePath : String
, parameterTypeService: ParameterTypeService
, techniqueSerializer : TechniqueSerializer
, doRudderLangTest : Boolean
, compiler : RudderCRunner
, errorLogPath : String
) {

private[this] val agentSpecific = new ClassicTechniqueWriter(basePath, parameterTypeService) :: new DSCTechniqueWriter(basePath, translater, parameterTypeService) :: Nil
Expand Down Expand Up @@ -178,7 +211,7 @@ class TechniqueWriter (
}
}

def techniqueMetadataContent(technique : Technique, methods: Map[BundleName, GenericMethod]) : PureResult[XmlNode] = {
def techniqueMetadataContent(technique : EditorTechnique, methods: Map[BundleName, GenericMethod]) : PureResult[XmlNode] = {

def reportingValuePerMethod (component: String, calls :Seq[MethodCall]) : PureResult[Seq[XmlNode]] = {

Expand Down Expand Up @@ -263,48 +296,53 @@ class TechniqueWriter (
}
}

def writeTechniqueAndUpdateLib(technique : Technique, methods: Map[BundleName, GenericMethod], modId : ModificationId, committer : EventActor) : IOResult[Technique] = {
def writeTechniqueAndUpdateLib(technique : EditorTechnique, methods: Map[BundleName, GenericMethod], modId : ModificationId, committer : EventActor) : IOResult[EditorTechnique] = {
for {
updatedTechnique <- writeTechnique(technique,methods,modId,committer)
libUpdate <- techLibUpdate.update(modId, committer, Some(s"Update Technique library after creating files for ncf Technique ${technique.name}")).
toIO.chainError(s"An error occured during technique update after files were created for ncf Technique ${technique.name}")
_ <- ZIO.when(doRudderLangTest) {
IOResult.effect(runRudderLangTestLoop(technique)).catchAll(err =>
ApplicationLoggerPure.error("Error when doing rudder-lang test loop. You can disable that test with property " +
s"'rudder.lang.test-loop.exec' in rudder config file: ${err.fullMsg}"))
}
} yield {
updatedTechnique
}
}

def runRudderLangTestLoop(technique : Technique): Unit = {
System.getenv().asScala.get("PATH") match {
case Some(path: String) => RunNuCommand.run(Cmd("/opt/rudder/share/rudder-lang/tools/tester.sh", technique.bundleName.value :: technique.category :: Nil, Map("PATH" -> path))).either.runNow match {
case Left(error: Error) => ApplicationLogger.error(error.getMessage)
case Left(error: RudderError) => ApplicationLogger.error(error.msg)
case Right(_) => ApplicationLogger.info(s"rudder-lang tester successfully looped for technique ${technique.bundleName.value}")
}
case None => ApplicationLogger.error(s"PATH environment variable must be defined to run rudder-lang test loop.")
}
}

// Write and commit all techniques files
def writeTechnique(technique : Technique, methods: Map[BundleName, GenericMethod], modId : ModificationId, committer : EventActor) : IOResult[Technique] = {
def writeTechnique(technique : EditorTechnique, methods: Map[BundleName, GenericMethod], modId : ModificationId, committer : EventActor) : IOResult[EditorTechnique] = {
for {
agentFiles <- writeAgentFiles(technique, methods, modId, committer)
metadata <- writeMetadata(technique, methods, modId, committer)
// Before writing down technique, set all resources to Untouched state, and remove Delete resources, was the cause of #17750
updateResources = technique.ressources.collect{case r if r.state != ResourceFileState.Deleted => r.copy(state = ResourceFileState.Untouched) }
techniqueWithResourceUpdated = technique.copy(ressources = updateResources)
updateResources = technique.ressources.collect{case r if r.state != ResourceFileState.Deleted => r.copy(state = ResourceFileState.Untouched) }
techniqueWithResourceUpdated = technique.copy(ressources = updateResources)
json <- writeJson(techniqueWithResourceUpdated)
agentFiles <- compiler.compileTechnique(technique).catchAll { e =>
val errorPath : File = root / errorLogPath / "rudderc" / "failures" / s"${DateTime.now()}_${technique.bundleName.value}.log"
for {
_ <- ApplicationLoggerPure.error(s"An error occurred when compiling technique '${technique.name}' (id : '${technique.bundleName}') with rudderc, error details in ${errorPath}, falling back to old saving process")
_ <- IOResult.effect {
errorPath.createFileIfNotExists(true)
errorPath.write(
s"""
|error =>
| ${e.fullMsg}
|technique data =>
| ${net.liftweb.json.prettyRender(techniqueSerializer.serializeTechniqueMetadata(technique))}""".stripMargin)
}.catchAll(e2 =>
ApplicationLoggerPure.error(s"Error when writing error log of '${technique.name}' (id : '${technique.bundleName}') in in ${errorPath}: ${e2.fullMsg}") *>
ApplicationLoggerPure.error(s"Error when compiling '${technique.name}' (id : '${technique.bundleName}') with rudderc was: ${e.fullMsg}")
)
_ <- writeAgentFiles(technique, methods, modId, committer)
} yield {
()
}
}

commit <- archiver.commitTechnique(technique, modId, committer, s"Committing technique ${technique.name}")
} yield {
techniqueWithResourceUpdated
}
}

def writeAgentFiles(technique : Technique, methods: Map[BundleName, GenericMethod], modId : ModificationId, commiter : EventActor) : IOResult[Seq[String]] = {
def writeAgentFiles(technique : EditorTechnique, methods: Map[BundleName, GenericMethod], modId : ModificationId, commiter : EventActor) : IOResult[Seq[String]] = {
for {
// Create/update agent files, filter None by flattening to list
files <- ZIO.foreach(agentSpecific)(_.writeAgentFiles(technique, methods)).map(_.flatten)
Expand All @@ -313,7 +351,7 @@ class TechniqueWriter (
}
}

def writeMetadata(technique : Technique, methods: Map[BundleName, GenericMethod], modId : ModificationId, commiter : EventActor) : IOResult[String] = {
def writeMetadata(technique : EditorTechnique, methods: Map[BundleName, GenericMethod], modId : ModificationId, commiter : EventActor) : IOResult[String] = {

val metadataPath = s"techniques/${technique.category}/${technique.bundleName.value}/${technique.version.value}/metadata.xml"

Expand All @@ -330,9 +368,8 @@ class TechniqueWriter (
}
}


def writeJson(technique: Technique) = {
val metadataPath = s"techniques/${technique.category}/${technique.bundleName.value}/${technique.version.value}/technique.json"
def writeJson(technique: EditorTechnique) = {
val metadataPath = s"${technique.path}/technique.json"

val path = s"${basePath}/${metadataPath}"

Expand All @@ -352,16 +389,16 @@ class TechniqueWriter (

trait AgentSpecificTechniqueWriter {

def writeAgentFiles( technique : Technique, methods : Map[BundleName, GenericMethod] ) : IOResult[Seq[String]]
def writeAgentFiles( technique : EditorTechnique, methods : Map[BundleName, GenericMethod] ) : IOResult[Seq[String]]

def agentMetadata ( technique : Technique, methods : Map[BundleName, GenericMethod] ) : PureResult[NodeSeq]
def agentMetadata ( technique : EditorTechnique, methods : Map[BundleName, GenericMethod] ) : PureResult[NodeSeq]
}

class ClassicTechniqueWriter(basePath : String, parameterTypeService: ParameterTypeService) extends AgentSpecificTechniqueWriter {

// We need to add a reporting bundle for this method to generate a na report for any method with a condition != any/cfengine (which ~= true
def methodNeedReporting(call : MethodCall, method : GenericMethod) = (! method.agentSupport.contains(AgentType.CfeCommunity)) || call.condition != "any" && call.condition != "cfengine-community"
def needReportingBundle(technique : Technique, methods : Map[BundleName, GenericMethod]) = technique.methodCalls.exists(c => methods.get(c.methodId).map(m => methodNeedReporting(c,m)).getOrElse(true))
def needReportingBundle(technique : EditorTechnique, methods : Map[BundleName, GenericMethod]) = technique.methodCalls.exists(c => methods.get(c.methodId).map(m => methodNeedReporting(c,m)).getOrElse(true))

def canonifyCondition(methodCall: MethodCall) = {
methodCall.condition.replaceAll("""(\$\{[^\}]*})""","""",canonify("$1"),"""")
Expand All @@ -377,7 +414,7 @@ class ClassicTechniqueWriter(basePath : String, parameterTypeService: ParameterT



def writeAgentFiles( technique : Technique, methods : Map[BundleName, GenericMethod] ) : IOResult[Seq[String]] = {
def writeAgentFiles( technique : EditorTechnique, methods : Map[BundleName, GenericMethod] ) : IOResult[Seq[String]] = {

val bundleParams = if (technique.parameters.nonEmpty) technique.parameters.map(_.name.canonify).mkString("(",",",")") else ""

Expand Down Expand Up @@ -512,7 +549,7 @@ class ClassicTechniqueWriter(basePath : String, parameterTypeService: ParameterT
tech +: repo
}
}
def agentMetadata ( technique : Technique, methods : Map[BundleName, GenericMethod] ) : PureResult[NodeSeq] = {
def agentMetadata ( technique : EditorTechnique, methods : Map[BundleName, GenericMethod] ) : PureResult[NodeSeq] = {
// We need to add reporting bundle if there is a method call that does not support cfengine (agent support does not contains both cfe agent)
val noAgentSupportReporting = technique.methodCalls.exists( m =>
methods.get(m.methodId).exists( gm =>
Expand Down Expand Up @@ -561,10 +598,10 @@ class DSCTechniqueWriter(
val genericParams =
"-reportId $reportId -techniqueName $techniqueName -auditOnly:$auditOnly"

def computeTechniqueFilePath(technique : Technique) =
def computeTechniqueFilePath(technique : EditorTechnique) =
s"techniques/${technique.category}/${technique.bundleName.value}/${technique.version.value}/technique.ps1"

def writeAgentFiles(technique : Technique, methods : Map[BundleName, GenericMethod] ): IOResult[Seq[String]] = {
def writeAgentFiles(technique : EditorTechnique, methods : Map[BundleName, GenericMethod] ): IOResult[Seq[String]] = {

def toDscFormat(call : MethodCall) : PureResult[String]= {

Expand Down Expand Up @@ -698,7 +735,7 @@ class DSCTechniqueWriter(
}
}

def agentMetadata(technique : Technique, methods : Map[BundleName, GenericMethod] ) = {
def agentMetadata(technique : EditorTechnique, methods : Map[BundleName, GenericMethod] ) = {
val xml = <AGENT type="dsc">
<BUNDLES>
<NAME>{technique.bundleName.validDscName}</NAME>
Expand Down Expand Up @@ -726,7 +763,7 @@ class DSCTechniqueWriter(

trait TechniqueArchiver {
def deleteTechnique(techniqueName : String, techniqueVersion : String, category : String, modId: ModificationId, commiter: EventActor, msg : String) : IOResult[Unit]
def commitTechnique(technique : Technique, modId: ModificationId, commiter: EventActor, msg : String) : IOResult[Unit]
def commitTechnique(technique : EditorTechnique, modId: ModificationId, commiter: EventActor, msg : String) : IOResult[Unit]
}

class TechniqueArchiverImpl (
Expand All @@ -752,7 +789,7 @@ class TechniqueArchiverImpl (
}).chainError(s"error when deleting and committing Technique '${techniqueName}/${techniqueVersion}").unit
}

def commitTechnique(technique : Technique, modId: ModificationId, commiter: EventActor, msg : String) : IOResult[Unit] = {
def commitTechnique(technique : EditorTechnique, modId: ModificationId, commiter: EventActor, msg : String) : IOResult[Unit] = {

val techniqueGitPath = s"techniques/${technique.category}/${technique.bundleName.value}/${technique.version.value}"
val filesToAdd = (
Expand All @@ -761,6 +798,7 @@ class TechniqueArchiverImpl (
"technique.cf" +:
"technique.ps1" +:
"technique.json" +:
"technique.rl" +:
technique.ressources.collect {
case ResourceFile(path, action) if action == ResourceFileState.New | action == ResourceFileState.Modified =>
s"resources/${path}"
Expand Down
12 changes: 7 additions & 5 deletions ...tion/rudder/ncf/TestTechniqueWriter.scala → ...udder/ncf/TestEditorTechniqueWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ import org.apache.commons.io.FileUtils
import org.specs2.specification.BeforeAfterAll

@RunWith(classOf[JUnitRunner])
class TestTechniqueWriter extends Specification with ContentMatchers with Loggable with BeforeAfterAll {
class TestEditorTechniqueWriter extends Specification with ContentMatchers with Loggable with BeforeAfterAll {
sequential
lazy val basePath = "/tmp/test-technique-writer-" + DateTime.now.toString()

Expand All @@ -116,7 +116,7 @@ class TestTechniqueWriter extends Specification with ContentMatchers with Loggab

val expectedPath = "src/test/resources/configuration-repository"
object TestTechniqueArchiver extends TechniqueArchiver {
def commitTechnique(technique : Technique, modId: ModificationId, commiter: EventActor, msg : String) : IOResult[Unit] = UIO.unit
def commitTechnique(technique : EditorTechnique, modId: ModificationId, commiter: EventActor, msg : String) : IOResult[Unit] = UIO.unit
def deleteTechnique(techniqueName: String, techniqueVersion: String, category : String, modId: ModificationId, commiter: EventActor, msg: String): IOResult[Unit] = UIO.unit
}

Expand Down Expand Up @@ -221,6 +221,7 @@ class TestTechniqueWriter extends Specification with ContentMatchers with Loggab

val valueCompiler = new InterpolatedValueCompilerImpl
val parameterTypeService : PlugableParameterTypeService = new PlugableParameterTypeService
val techniqueCompiler = new RudderCRunner("/opt/rudder/etc/rudderc.conf","/opt/rudder/bin/rudderc",basePath)
val writer = new TechniqueWriter(
TestTechniqueArchiver
, TestLibUpdater
Expand All @@ -233,7 +234,8 @@ class TestTechniqueWriter extends Specification with ContentMatchers with Loggab
, basePath
, parameterTypeService
, new TechniqueSerializer(parameterTypeService)
, false
, techniqueCompiler
, basePath
)
val dscWriter = new DSCTechniqueWriter(basePath, valueCompiler, new ParameterType.PlugableParameterTypeService)
val classicWriter = new ClassicTechniqueWriter(basePath, new ParameterType.PlugableParameterTypeService)
Expand Down Expand Up @@ -307,7 +309,7 @@ class TestTechniqueWriter extends Specification with ContentMatchers with Loggab
Nil ).map(m => (m.id,m)).toMap

val technique =
Technique(
EditorTechnique(
BundleName("technique_by_Rudder")
, "Test Technique created through Rudder API"
, "ncf_techniques"
Expand Down Expand Up @@ -405,7 +407,7 @@ class TestTechniqueWriter extends Specification with ContentMatchers with Loggab
}

val technique_any =
Technique(
EditorTechnique(
BundleName("technique_any")
, "Test Technique created through Rudder API"
, "ncf_techniques"
Expand Down
2 changes: 1 addition & 1 deletion .../sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/ncf/TechniqueReader.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class TechniqueReader(
result
}
}
def readTechniquesMetadataFile: IOResult[List[Technique]] = {
def readTechniquesMetadataFile: IOResult[List[EditorTechnique]] = {
for {
methods <- readMethodsMetadataFile
techniqueFiles <- getAllTechniqueFiles(configuration_repository / "techniques")
Expand Down
Loading