From 54260842b4843ec255396c74aad72a096e293aa2 Mon Sep 17 00:00:00 2001 From: "Francois @fanf42 Armand" Date: Wed, 31 May 2017 13:33:14 +0200 Subject: [PATCH] Fixes #10823: Select system techniques and generate correct policies based on agent type --- .../normation/cfclerk/domain/Technique.scala | 27 +- .../cfclerk/domain/TechniqueResource.scala | 9 +- .../services/impl/GitTechniqueReader.scala | 6 +- .../cfclerk/xmlparsers/TechniqueParser.scala | 138 +++-- .../services/policies/DeploymentService.scala | 6 +- .../rudder/services/policies/RuleVal.scala | 13 +- .../nodeconfig/NodeConfiguration.scala | 12 +- .../NodeConfigurationCacheRepository.scala | 29 +- .../nodeconfig/NodeConfigurationLogger.scala | 17 +- .../nodeconfig/NodeConfigurationService.scala | 5 +- .../NodeConfigurationServiceImpl.scala | 15 +- .../policies/write/AgentSpecificLogic.scala | 192 +++++++ .../policies/write/BuildBundleSequence.scala | 483 +++++++++++------- .../policies/write/PathComputer.scala | 2 - ...ervice.scala => PolicyWriterService.scala} | 123 ++--- .../write/PrepareTemplateVariables.scala | 69 +-- .../write/PromiseWriteDataStructures.scala | 3 +- .../rules/dsc/Create_file/1.0/Create_file.ps1 | 4 + .../dsc/dsc-agent/1.0/some-resource.conf | 3 + .../rules/dsc/properties.d/properties.json | 3 + .../rules/dsc/rudder-directives.ps1 | 27 + .../rules/dsc/rudder-system-directives.ps1 | 39 ++ .../rules/dsc/rudder.json | 1 + .../rules/dsc/rudder.ps1 | 8 + .../common/1.0/cf-served.cf | 4 +- .../distributePolicy/1.0/nodeslist.json | 2 +- .../common/1.0/cf-served.cf | 4 +- .../distributePolicy/1.0/nodeslist.json | 2 +- .../techniques/dsc-agent/category.xml | 25 + .../dsc-agent/dsc-agent/1.0/metadata.xml | 58 +++ .../dsc-agent/1.0/rudder-directives.st | 27 + .../dsc-agent/1.0/rudder-system-directives.st | 39 ++ .../dsc-agent/dsc-agent/1.0/rudder.ps1 | 8 + .../dsc-agent/1.0/some-resource.conf | 3 + .../Create_file/1.0/Create_file.ps1 | 4 + .../Create_file/1.0/metadata.xml | 13 +- .../cfclerk/domain/TechniqueTest.scala | 18 +- .../services/DummyTechniqueRepository.scala | 15 +- .../services/JGitPackageReaderTest.scala | 4 +- .../services/policies/NodeConfigData.scala | 85 ++- .../policies/RuleValServiceTest.scala | 4 +- .../write/PolicyInstanceAgregationTest.scala | 16 +- .../write/PrepareTemplateVariableTest.scala | 22 +- .../write/WriteSystemTechniquesTest.scala | 161 ++++-- .../scala/bootstrap/liftweb/AppConfig.scala | 7 +- 45 files changed, 1253 insertions(+), 502 deletions(-) create mode 100644 rudder-core/src/main/scala/com/normation/rudder/services/policies/write/AgentSpecificLogic.scala rename rudder-core/src/main/scala/com/normation/rudder/services/policies/write/{Cf3PromisesFileWriterService.scala => PolicyWriterService.scala} (92%) create mode 100644 rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/Create_file/1.0/Create_file.ps1 create mode 100644 rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/dsc-agent/1.0/some-resource.conf create mode 100644 rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/properties.d/properties.json create mode 100644 rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-directives.ps1 create mode 100644 rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-system-directives.ps1 create mode 100644 rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.json create mode 100644 rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.ps1 create mode 100644 rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/category.xml create mode 100644 rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/metadata.xml create mode 100644 rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-directives.st create mode 100644 rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-system-directives.st create mode 100644 rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder.ps1 create mode 100644 rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/some-resource.conf create mode 100644 rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/Create_file.ps1 diff --git a/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala b/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala index c906e093eb..e61f88e20f 100644 --- a/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala +++ b/rudder-core/src/main/scala/com/normation/cfclerk/domain/Technique.scala @@ -39,6 +39,7 @@ package com.normation.cfclerk.domain import com.normation.utils.Utils._ import com.normation.utils.HashcodeCaching +import com.normation.inventory.domain.AgentType /** @@ -69,6 +70,13 @@ case class TechniqueId(name: TechniqueName, version: TechniqueVersion) extends O } } +final case class AgentConfig( + agentType: AgentType + , templates : Seq[TechniqueTemplate] + , files : Seq[TechniqueFile] + , bundlesequence : Seq[BundleName] +) + /** * A structure containing all informations about a technique deprecation */ @@ -86,9 +94,7 @@ case class Technique( id : TechniqueId , name : String , description : String - , templates : Seq[TechniqueTemplate] - , files : Seq[TechniqueFile] - , bundlesequence : Seq[Bundle] + , agentConfigs : List[AgentConfig] , trackerVariableSpec : TrackerVariableSpec , rootSection : SectionSpec //be careful to not split it from the TechniqueId, else you will not have the good spec for the version , deprecrationInfo : Option[TechniqueDeprecationInfo] @@ -105,17 +111,10 @@ case class Technique( require(nonEmpty(name), "Name is required in policy") /** - * Utity method that retrieve the map of all template full name for that policy + * Utity method that retrieve all templates IDs + * Be carefull, you will get all templates for all agents */ - val templatesMap: Map[TechniqueResourceId, TechniqueTemplate] = templates.map(t => (t.id, t)).toMap - - def toLongString: String = { - "## %s [%s-%s] ## \n -> unique:%-5s \n -> %s\n -> templates: %s".format( - name, id.name.value, id.version.toString, - if (isMultiInstance) "false" else "true", - description, - templates.mkString(" : ")) - } + val templatesIds: Set[TechniqueResourceId] = agentConfigs.flatMap(cfg => cfg.templates.map(_.id)).toSet val getAllVariableSpecs = this.rootSection.getAllVariables ++ this.systemVariableSpecs :+ this.trackerVariableSpec } @@ -124,7 +123,7 @@ case class Technique( /** * The representation of a bundle name, used for the bundlesequence */ -case class Bundle(name : String) extends HashcodeCaching +case class BundleName(value : String) extends HashcodeCaching object Technique { def normalizeName(name: String): String = { diff --git a/rudder-core/src/main/scala/com/normation/cfclerk/domain/TechniqueResource.scala b/rudder-core/src/main/scala/com/normation/cfclerk/domain/TechniqueResource.scala index ec1d5d8b7a..b709faa9d3 100644 --- a/rudder-core/src/main/scala/com/normation/cfclerk/domain/TechniqueResource.scala +++ b/rudder-core/src/main/scala/com/normation/cfclerk/domain/TechniqueResource.scala @@ -37,7 +37,6 @@ package com.normation.cfclerk.domain -import com.normation.utils.HashcodeCaching /** * Representation of a technique resource id. @@ -45,9 +44,15 @@ import com.normation.utils.HashcodeCaching * A resource may be either defined relatively to a technique, or relatively to * the configuration-repository directory (the git root directory). * Template filename extension is mandatory to be ".st". + * + * Agent type is not part of the ID because: + * - for a template, extension must be .st, so if you want different template for + * cfengine and something else, you must have different base name. + * - for a "by path" resources, the path must be different for the resources to be + * different. */ sealed trait TechniqueResourceId { - def name: String + def name: String } /** diff --git a/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala b/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala index 29631ebd9b..d8983763bd 100644 --- a/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala +++ b/rudder-core/src/main/scala/com/normation/cfclerk/services/impl/GitTechniqueReader.scala @@ -614,7 +614,7 @@ class GitTechniqueReader( private[this] val dummyTechnique = Technique( TechniqueId(TechniqueName("dummy"),TechniqueVersion("1.0")) - , "dummy", "dummy", Seq(), Seq(), Seq(), TrackerVariableSpec() + , "dummy", "dummy", Nil, TrackerVariableSpec() , SectionSpec("ROOT"), None ) @@ -781,8 +781,8 @@ class GitTechniqueReader( //also add template "by path" val techniques = techniqueInfos.techniques.flatMap { case(_, set) => set.map { case(_, t) => t } } techniques.foreach { t => - val byPath = t.templates.collect { case TechniqueTemplate(id@TechniqueResourceIdByPath(_,_),_,_) => id } ++ - t.files.collect { case TechniqueFile(id@TechniqueResourceIdByPath(_,_),_,_) => id } + val byPath = t.agentConfigs.flatMap(cfg => cfg.templates.collect { case TechniqueTemplate(id@TechniqueResourceIdByPath(_,_),_,_) => id }) ++ + t.agentConfigs.flatMap(cfg => cfg.files.collect { case TechniqueFile(id@TechniqueResourceIdByPath(_,_),_,_) => id }) byPath.foreach { resource => //here, "/" is needed at the begining because diffEntry have one, so if we don't //add it, we won't find is back in modifiedTechnique and diffPathEntries diff --git a/rudder-core/src/main/scala/com/normation/cfclerk/xmlparsers/TechniqueParser.scala b/rudder-core/src/main/scala/com/normation/cfclerk/xmlparsers/TechniqueParser.scala index 351d0fdd13..00790ec5c4 100644 --- a/rudder-core/src/main/scala/com/normation/cfclerk/xmlparsers/TechniqueParser.scala +++ b/rudder-core/src/main/scala/com/normation/cfclerk/xmlparsers/TechniqueParser.scala @@ -44,6 +44,7 @@ import com.normation.cfclerk.exceptions.ParsingException import com.normation.utils.Utils._ import scala.xml._ import net.liftweb.common._ +import com.normation.inventory.domain.AgentType /** * Parse a technique (metadata.xml file) @@ -55,46 +56,54 @@ class TechniqueParser( , systemVariableSpecService : SystemVariableSpecService ) extends Loggable { - def parseXml(node: Node, id: TechniqueId, expectedReportCsvExists: Boolean): Technique = { - //check that node is and has a name attribute - if (node.label.toUpperCase == TECHNIQUE_ROOT) { - node.attribute(TECHNIQUE_NAME) match { + def parseXml(xml: Node, id: TechniqueId, expectedReportCsvExists: Boolean): Technique = { + //check that xml is and has a name attribute + if (xml.label.toUpperCase == TECHNIQUE_ROOT) { + xml.attribute(TECHNIQUE_NAME) match { case Some(nameAttr) if (TechniqueParser.isValidId(id.name.value) && nonEmpty(nameAttr.text)) => val name = nameAttr.text - val compatible = try Some(parseCompatibleTag((node \ COMPAT_TAG).head)) catch { case _:Exception => None } + val compatible = try Some(parseCompatibleTag((xml \ COMPAT_TAG).head)) catch { case _:Exception => None } - val rootSection = sectionSpecParser.parseSectionsInPolicy(node, id, name) + val rootSection = sectionSpecParser.parseSectionsInPolicy(xml, id, name) - val description = ??!((node \ TECHNIQUE_DESCRIPTION).text).getOrElse(name) + val description = ??!((xml \ TECHNIQUE_DESCRIPTION).text).getOrElse(name) - val templates = (node \ PROMISE_TEMPLATES_ROOT \\ PROMISE_TEMPLATE).map(xml => parseTemplate(id, xml) ) - val files = (node \ FILES \\ FILE).map(xml => parseFile(id, xml) ) + val trackerVariableSpec = parseTrackerVariableSpec(xml) - val bundlesequence = (node \ BUNDLES_ROOT \\ BUNDLE_NAME).map(xml => Bundle(xml.text) ) + val systemVariableSpecs = parseSysvarSpecs(xml,id) - val trackerVariableSpec = parseTrackerVariableSpec(node) + val isMultiInstance = ((xml \ TECHNIQUE_IS_MULTIINSTANCE).text.equalsIgnoreCase("true") ) - val systemVariableSpecs = parseSysvarSpecs(node,id) + val longDescription = ??!((xml \ TECHNIQUE_LONG_DESCRIPTION).text).getOrElse("") - val isMultiInstance = ((node \ TECHNIQUE_IS_MULTIINSTANCE).text.equalsIgnoreCase("true") ) - - val longDescription = ??!((node \ TECHNIQUE_LONG_DESCRIPTION).text).getOrElse("") - - val isSystem = ((node \ TECHNIQUE_IS_SYSTEM).text.equalsIgnoreCase("true")) + val isSystem = ((xml \ TECHNIQUE_IS_SYSTEM).text.equalsIgnoreCase("true")) //the technique provides its expected reports if at least one section has a variable of type REPORT_KEYS val providesExpectedReports = expectedReportCsvExists - val deprecationInfo = parseDeprecrationInfo(node) + val deprecationInfo = parseDeprecrationInfo(xml) + + val agentConfigs = ( + (xml \ "AGENT" ).toList.map(agent => parseAgentConfig(id, agent)).flatten ++ + { + //for compability reason, we cheat and make the template/file/bundlesequence under root + //and not in an sub-element be considered as being in + val forCompatibilityAgent = + {(xml \ PROMISE_TEMPLATES_ROOT)} + {(xml \ FILES)} + {(xml \ BUNDLES_ROOT)} + + + parseAgentConfig(id, forCompatibilityAgent) + } + ) val technique = Technique( id , name , description - , templates - , files - , bundlesequence + , agentConfigs , trackerVariableSpec , rootSection , deprecationInfo @@ -123,15 +132,48 @@ class TechniqueParser( technique - case _ => throw new ParsingException("Not a policy node, missing 'name' attribute: %s".format(node)) + case _ => throw new ParsingException("Not a policy xml, missing 'name' attribute: %s".format(xml)) } } else { - throw new ParsingException("Not a policy node, bad node name. Was expecting <%s>, got: %s".format(TECHNIQUE_ROOT,node)) + throw new ParsingException("Not a policy xml, bad xml name. Was expecting <%s>, got: %s".format(TECHNIQUE_ROOT,xml)) + } + } + + /* + * Here, we are parsing xml, that contains the list of templates/files/bundles + * defined for the given agent. + * + * id is for reporting + */ + private[this] def parseAgentConfig(id: TechniqueId, xml: Node): List[AgentConfig] = { + //start to parse agent types for that config. It's a comma separated list + import scala.language.postfixOps + + if(xml.label != "AGENT") { + Nil + } else { + + val agentTypes = (xml \ "@type" text).split(",").map { name => + AgentType.fromValue(name) match { + case Full(agentType) => Some(agentType) + case eb: EmptyBox => + val msg = s"Error when parsing technique with id '${id.toString}', agent type='${name}' is not known and the corresponding config will be ignored" + val e = eb ?~! msg + logger.warn(e.messageChain) + None + } + }.flatten.toList + + val templates = (xml \ PROMISE_TEMPLATES_ROOT \\ PROMISE_TEMPLATE).map(xml => parseTemplate(id, xml) ) + val files = (xml \ FILES \\ FILE).map(xml => parseFile(id, xml) ) + val bundlesequence = (xml \ BUNDLES_ROOT \\ BUNDLE_NAME).map(xml => BundleName(xml.text) ) + + agentTypes.map( agentType => AgentConfig(agentType, templates, files, bundlesequence)) } } - private[this] def parseTrackerVariableSpec(node: Node): TrackerVariableSpec = { - val trackerVariableSpecs = (node \ TRACKINGVAR) + private[this] def parseTrackerVariableSpec(xml: Node): TrackerVariableSpec = { + val trackerVariableSpecs = (xml \ TRACKINGVAR) if(trackerVariableSpecs.size == 0) { //default trackerVariable variable spec for that package TrackerVariableSpec() } else if(trackerVariableSpecs.size == 1) { @@ -143,9 +185,9 @@ class TechniqueParser( } else throw new ParsingException("Only one <%s> tag is allowed the the document, but found %s".format(TRACKINGVAR,trackerVariableSpecs.size)) } - private[this] def parseDeprecrationInfo(node: Node): Option[TechniqueDeprecationInfo] = { + private[this] def parseDeprecrationInfo(xml: Node): Option[TechniqueDeprecationInfo] = { for { - deprecationInfo <- (node \ TECHNIQUE_DEPRECATION_INFO).headOption + deprecationInfo <- (xml \ TECHNIQUE_DEPRECATION_INFO).headOption } yield { val message = deprecationInfo.text if (message.size == 0) { @@ -161,8 +203,8 @@ class TechniqueParser( * Parse the list of system vars used by that policy package. * */ - private[this] def parseSysvarSpecs(node: Node, id:TechniqueId) : Set[SystemVariableSpec] = { - (node \ SYSTEMVARS_ROOT \ SYSTEMVAR_NAME).map{ x => + private[this] def parseSysvarSpecs(xml: Node, id:TechniqueId) : Set[SystemVariableSpec] = { + (xml \ SYSTEMVARS_ROOT \ SYSTEMVAR_NAME).map{ x => try { systemVariableSpecService.get(x.text) } catch { @@ -190,7 +232,7 @@ class TechniqueParser( * to root of configuration repository in place of relative to the technique. * */ - private[this] def parseResource(techniqueId: TechniqueId, node: Node, isTemplate:Boolean): (TechniqueResourceId, String) = { + private[this] def parseResource(techniqueId: TechniqueId, xml: Node, isTemplate:Boolean): (TechniqueResourceId, String) = { def fileToList(f: java.io.File): List[String] = { if(f == null) { @@ -203,17 +245,17 @@ class TechniqueParser( //the default out path for a template with name "name" is "techniqueName/techniqueVersion/name.cf def defaultOutPath(name: String) = s"${techniqueId.name.value}/${techniqueId.version.toString}/${name}${if(isTemplate) TechniqueTemplate.promiseExtension else ""}" - val outPath = (node \ PROMISE_TEMPLATE_OUTPATH).text match { + val outPath = (xml \ PROMISE_TEMPLATE_OUTPATH).text match { case "" => None case path => Some(path) } - val id = node.attribute(PROMISE_TEMPLATE_NAME) match { + val id = xml.attribute(PROMISE_TEMPLATE_NAME) match { case Some(attr) if (attr.size == 1) => // some checking on name val n = attr.text.trim if(n.startsWith("/") || n.endsWith("/")) { - throw new ParsingException(s"Error when parsing xml ${node}. Resource name must not start nor end with '/'") + throw new ParsingException(s"Error when parsing xml ${xml}. Resource name must not start nor end with '/'") } else { if(n.startsWith(RUDDER_CONFIGURATION_REPOSITORY+"/")) { @@ -231,7 +273,7 @@ class TechniqueParser( } } - case _ => throw new ParsingException(s"Error when parsing xml ${node}. Resource name is not defined") + case _ => throw new ParsingException(s"Error when parsing xml ${xml}. Resource name is not defined") } (id, outPath.getOrElse(defaultOutPath(id.name))) } @@ -239,36 +281,36 @@ class TechniqueParser( /** * A file is almost exactly like a Template, safe the include that we don't care of. */ - def parseFile(techniqueId: TechniqueId, node: Node): TechniqueFile = { - if(node.label != FILE) throw new ParsingException(s"Error: try to parse a <${FILE}> node, but actually get: ${node}") + def parseFile(techniqueId: TechniqueId, xml: Node): TechniqueFile = { + if(xml.label != FILE) throw new ParsingException(s"Error: try to parse a <${FILE}> xml, but actually get: ${xml}") // Default value for FILE is false, so we should only check if the value is true and if it is empty it - val included = (node \ PROMISE_TEMPLATE_INCLUDED).text == "true" - val (id, out) = parseResource(techniqueId, node, false) + val included = (xml \ PROMISE_TEMPLATE_INCLUDED).text == "true" + val (id, out) = parseResource(techniqueId, xml, false) TechniqueFile(id, out, included) } - def parseTemplate(techniqueId: TechniqueId, node: Node): TechniqueTemplate = { - if(node.label != PROMISE_TEMPLATE) throw new ParsingException(s"Error: try to parse a <${PROMISE_TEMPLATE}> node, but actually get: ${node}") - val included = !((node \ PROMISE_TEMPLATE_INCLUDED).text == "false") - val (id, out) = parseResource(techniqueId, node, true) + def parseTemplate(techniqueId: TechniqueId, xml: Node): TechniqueTemplate = { + if(xml.label != PROMISE_TEMPLATE) throw new ParsingException(s"Error: try to parse a <${PROMISE_TEMPLATE}> xml, but actually get: ${xml}") + val included = !((xml \ PROMISE_TEMPLATE_INCLUDED).text == "false") + val (id, out) = parseResource(techniqueId, xml, true) TechniqueTemplate(id, out, included) } /** * Parse a marker - * @param node example : + * @param xml example : * * Ubuntu * debian-5 * cfengine-community * - * @return A compatible variable corresponding to the entry node + * @return A compatible variable corresponding to the entry xml */ - def parseCompatibleTag(node: Node): Compatible = { - if(node.label != COMPAT_TAG) throw new ParsingException("CompatibleParser was expecting a <%s> node and get:\n%s".format(COMPAT_TAG, node)) - val os = node \ COMPAT_OS map (n => + def parseCompatibleTag(xml: Node): Compatible = { + if(xml.label != COMPAT_TAG) throw new ParsingException("CompatibleParser was expecting a <%s> xml and get:\n%s".format(COMPAT_TAG, xml)) + val os = xml \ COMPAT_OS map (n => OperatingSystem(n.text, (n \ "@version").text)) - val agents = node \ COMPAT_AGENT map (n => + val agents = xml \ COMPAT_AGENT map (n => Agent(n.text, (n \ "@version").text)) Compatible(os, agents) } diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/DeploymentService.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/DeploymentService.scala index c006206b69..0e48fa434c 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/DeploymentService.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/DeploymentService.scala @@ -77,7 +77,7 @@ import com.normation.rudder.reports.AgentRunIntervalService import com.normation.rudder.reports.AgentRunInterval import com.normation.rudder.domain.logger.ComplianceDebugLogger import com.normation.rudder.services.reports.CachedFindRuleNodeStatusReports -import com.normation.rudder.services.policies.write.Cf3PromisesFileWriterService +import com.normation.rudder.services.policies.write.PolicyWriterService import com.normation.rudder.services.policies.write.Cf3PolicyDraft import com.normation.rudder.services.policies.write.Cf3PolicyDraftId import com.normation.rudder.reports.GlobalComplianceMode @@ -499,7 +499,7 @@ class PromiseGenerationServiceImpl ( , override val complianceModeService : ComplianceModeService , override val agentRunService : AgentRunIntervalService , override val complianceCache : CachedFindRuleNodeStatusReports - , override val promisesFileWriterService: Cf3PromisesFileWriterService + , override val promisesFileWriterService: PolicyWriterService , override val getAgentRunInterval: () => Box[Int] , override val getAgentRunSplaytime: () => Box[Int] , override val getAgentRunStartHour: () => Box[Int] @@ -859,7 +859,7 @@ trait PromiseGeneration_updateAndWriteRule extends PromiseGenerationService { def nodeConfigurationService : NodeConfigurationService def woRuleRepo: WoRuleRepository - def promisesFileWriterService: Cf3PromisesFileWriterService + def promisesFileWriterService: PolicyWriterService /** * That methode remove node configurations for nodes not in allNodes. diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/RuleVal.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/RuleVal.scala index 21f957d537..3896699479 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/RuleVal.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/RuleVal.scala @@ -37,22 +37,19 @@ package com.normation.rudder.services.policies -import scala.collection.immutable.TreeMap import com.normation.cfclerk.domain.Technique import com.normation.cfclerk.domain.TrackerVariable import com.normation.cfclerk.domain.Variable -import com.normation.inventory.domain.NodeInventory import com.normation.rudder.domain.nodes.NodeInfo import com.normation.rudder.domain.parameters.ParameterName -import com.normation.rudder.domain.reports.NodeAndConfigId -import com.normation.rudder.services.policies.write.Cf3PolicyDraft -import com.normation.rudder.services.policies.write.Cf3PolicyDraftId -import com.normation.utils.HashcodeCaching -import net.liftweb.common.Box import com.normation.rudder.domain.policies.DirectiveId +import com.normation.rudder.domain.policies.PolicyMode import com.normation.rudder.domain.policies.RuleId import com.normation.rudder.domain.policies.RuleTarget -import com.normation.rudder.domain.policies.PolicyMode +import com.normation.rudder.domain.reports.NodeAndConfigId +import com.normation.utils.HashcodeCaching +import net.liftweb.common.Box +import scala.collection.immutable.TreeMap final case class BundleOrder(value: String) diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfiguration.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfiguration.scala index eef3db233f..f30aa607c6 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfiguration.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfiguration.scala @@ -37,19 +37,17 @@ package com.normation.rudder.services.policies.nodeconfig -import org.joda.time.DateTime import com.normation.cfclerk.domain.TechniqueId import com.normation.cfclerk.domain.Variable import com.normation.rudder.domain.nodes.NodeInfo +import com.normation.rudder.domain.parameters.Parameter +import com.normation.rudder.domain.parameters.ParameterName import com.normation.rudder.domain.policies.RuleId +import com.normation.rudder.domain.reports.NodeModeConfig +import com.normation.rudder.services.policies.write.Cf3PolicyDraft +import com.normation.rudder.services.policies.write.Cf3PolicyDraftId import com.normation.utils.HashcodeCaching import net.liftweb.common.Loggable -import com.normation.rudder.domain.parameters.ParameterName -import com.normation.rudder.domain.parameters.Parameter -import com.normation.rudder.services.policies.write.ParameterEntry -import com.normation.rudder.services.policies.write.Cf3PolicyDraftId -import com.normation.rudder.services.policies.write.Cf3PolicyDraft -import com.normation.rudder.domain.reports.NodeModeConfig case class ParameterForConfiguration( diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationCacheRepository.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationCacheRepository.scala index aff4b7c6e8..f6a3705e50 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationCacheRepository.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationCacheRepository.scala @@ -37,22 +37,21 @@ package com.normation.rudder.services.policies.nodeconfig -import com.normation.rudder.domain.policies.RuleId +import com.normation.cfclerk.domain.Variable import com.normation.inventory.domain.NodeId -import net.liftweb.common.Box -import net.liftweb.common.Full -import org.joda.time.DateTime -import com.normation.rudder.domain.RudderDit -import com.normation.rudder.domain.RudderLDAPConstants.{OC_NODES_CONFIG, A_NODE_CONFIG} import com.normation.ldap.sdk.LDAPConnectionProvider -import com.normation.ldap.sdk.RwLDAPConnection import com.normation.ldap.sdk.LDAPEntry +import com.normation.ldap.sdk.RwLDAPConnection +import com.normation.rudder.domain.RudderDit +import com.normation.rudder.domain.RudderLDAPConstants.A_NODE_CONFIG +import com.normation.rudder.domain.RudderLDAPConstants.OC_NODES_CONFIG +import com.normation.rudder.services.policies.write.Cf3PolicyDraft +import com.normation.rudder.services.policies.write.Cf3PolicyDraftId +import net.liftweb.common.Box import net.liftweb.common.Failure +import net.liftweb.common.Full import net.liftweb.common.Loggable -import com.normation.cfclerk.domain.Variable -import com.normation.rudder.services.policies.write.Cf3PolicyDraftId -import com.normation.rudder.services.policies.write.Cf3PolicyDraft -import com.normation.rudder.domain.reports.NodeModeConfig +import org.joda.time.DateTime case class PolicyHash( @@ -144,14 +143,14 @@ object NodeConfigurationHash { */ val nodeInfoHashValue = { val i = nodeConfig.nodeInfo - List( + List[Int]( i.name.hashCode , i.hostname.hashCode , i.localAdministratorAccountName.hashCode , i.policyServerId.hashCode , i.properties.hashCode - , i.isPolicyServer - , i.serverRoles + , i.isPolicyServer.hashCode + , i.serverRoles.hashCode , i.agentsName.hashCode , nodeConfig.modesConfig.hashCode ).hashCode @@ -317,7 +316,7 @@ class LdapNodeConfigurationHashRepository( ) extends NodeConfigurationHashRepository with Loggable { import net.liftweb.json._ - import net.liftweb.json.Serialization.{read, write} + import net.liftweb.json.Serialization.{ read, write } implicit val formats = Serialization.formats(NoTypeHints) ++ net.liftweb.json.ext.JodaTimeSerializers.all /* diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationLogger.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationLogger.scala index ae4595f378..4afb0508ff 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationLogger.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationLogger.scala @@ -37,20 +37,16 @@ package com.normation.rudder.services.policies.nodeconfig -import java.io.File -import java.io.PrintWriter - -import org.joda.time.DateTime -import org.slf4j.LoggerFactory - import com.normation.inventory.domain.NodeId import com.normation.utils.Control._ - +import java.io.File +import java.io.PrintWriter import net.liftweb.common._ import net.liftweb.json.NoTypeHints import net.liftweb.json.Serialization import net.liftweb.json.Serialization.writePretty - +import org.joda.time.DateTime +import org.slf4j.LoggerFactory trait NodeConfigurationLogger { @@ -72,8 +68,6 @@ class NodeConfigurationLoggerImpl( val logger = LoggerFactory.getLogger("rudder.debug.nodeconfiguration") - import java.io.File - { val p = new File(path) p.mkdirs() @@ -86,10 +80,7 @@ class NodeConfigurationLoggerImpl( def log(nodeConfiguration: Seq[NodeConfiguration]): Box[Set[NodeId]] = { import net.liftweb.json._ - import net.liftweb.json.Serialization.writePretty implicit val formats = Serialization.formats(NoTypeHints) - import java.io.PrintWriter - def writeIn[T](path:File)(f: PrintWriter => Box[T]) = { val printWriter = new java.io.PrintWriter(path) try { diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationService.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationService.scala index 4fe1c9fa7c..a4226256c1 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationService.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationService.scala @@ -37,12 +37,9 @@ package com.normation.rudder.services.policies.nodeconfig -import com.normation.rudder.domain.servers._ -import net.liftweb.common.Box import com.normation.inventory.domain.NodeId -import com.normation.rudder.repository.FullActiveTechniqueCategory import com.normation.rudder.domain.policies.RuleId -import com.normation.rudder.domain.reports.NodeConfigId +import net.liftweb.common.Box import org.joda.time.DateTime trait NodeConfigurationService { diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationServiceImpl.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationServiceImpl.scala index 899be375b0..4b4986874c 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationServiceImpl.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/nodeconfig/NodeConfigurationServiceImpl.scala @@ -39,16 +39,10 @@ package com.normation.rudder.services.policies.nodeconfig import com.normation.inventory.domain.NodeId import com.normation.rudder.domain.policies.RuleId -import com.normation.rudder.services.policies.write.Cf3PolicyDraft -import com.normation.rudder.domain.reports.NodeConfigId -import com.normation.rudder.repository.FullActiveTechniqueCategory -import com.normation.utils.Control.sequence +import com.normation.rudder.services.policies.BundleOrder +import com.normation.utils.Control._ import net.liftweb.common._ import org.joda.time.DateTime -import com.normation.rudder.services.policies.write.Cf3PolicyDraft -import com.normation.rudder.services.policies.write.Cf3PolicyDraftId -import com.normation.rudder.services.policies.write.Cf3PromisesFileWriterService -import com.normation.rudder.services.policies.BundleOrder /** @@ -61,8 +55,7 @@ import com.normation.rudder.services.policies.BundleOrder * */ class NodeConfigurationServiceImpl( - policyTranslator : Cf3PromisesFileWriterService - , repository : NodeConfigurationHashRepository + repository: NodeConfigurationHashRepository ) extends NodeConfigurationService with Loggable { //delegate to repository for nodeconfig persistence @@ -74,7 +67,7 @@ class NodeConfigurationServiceImpl( def sanitize(targets : Seq[NodeConfiguration]) : Box[Map[NodeId, NodeConfiguration]] = { - /** + /* * Sanitize directive to the node configuration, returning a new node configuration with * updated directives. * diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/AgentSpecificLogic.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/AgentSpecificLogic.scala new file mode 100644 index 0000000000..ab9cb6936c --- /dev/null +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/AgentSpecificLogic.scala @@ -0,0 +1,192 @@ +/* +************************************************************************************* +* Copyright 2017 Normation SAS +************************************************************************************* +* +* This file is part of Rudder. +* +* Rudder is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* In accordance with the terms of section 7 (7. Additional Terms.) of +* the GNU General Public License version 3, the copyright holders add +* the following Additional permissions: +* Notwithstanding to the terms of section 5 (5. Conveying Modified Source +* Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General +* Public License version 3, when you create a Related Module, this +* Related Module is not considered as a part of the work and may be +* distributed under the license agreement of your choice. +* A "Related Module" means a set of sources files including their +* documentation that, without modification of the Source Code, enables +* supplementary functions or services in addition to those offered by +* the Software. +* +* Rudder is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with Rudder. If not, see . + +* +************************************************************************************* +*/ + +package com.normation.rudder.services.policies.write + +import org.apache.commons.io.FileUtils +import scala.io.Codec +import java.io.File +import net.liftweb.common.Box +import net.liftweb.util.Helpers.tryo +import com.normation.inventory.domain.AgentType +import com.normation.utils.Control.sequence +import net.liftweb.common.Full +import net.liftweb.common.Failure + +/* + * This file contain agent-type specific logic used during the policy + * writing process, mainly: + * - specific files (like expected_reports.csv for CFEngine-based agent) + * - specific format for "bundle" sequence. + */ + +//containser for agent specific file written during policy generation +final case class AgentSpecificFile( + path: String +) + +//how do we write bundle sequence / input files system variable for the agent? +trait AgentFormatBundleVariables { + import BuildBundleSequence._ + def getBundleVariables( + systemInputs: List[InputFile] + , sytemBundles: List[TechniqueBundles] + , userInputs : List[InputFile] + , userBundles : List[TechniqueBundles] + ) : BundleSequenceVariables +} + + +// does that implementation knows something about the current agent type +trait AgentSpecificGenerationHandle { + def handle(agentType: AgentType): Boolean +} + + +// specific generic (i.e non bundle order linked) system variable +// todo - need to plug in systemvariablespecservice +// idem for the bundle seq + +//write what must be written for the given configuration +trait WriteAgentSpecificFiles { + def write(cfg: AgentNodeWritableConfiguration): Box[List[AgentSpecificFile]] +} + + +// the pipeline of processing for the specific writes +object WriteAllAgentSpecificFiles extends WriteAgentSpecificFiles { + + /** + * Ordered list of handlers + */ + var pipeline: List[AgentSpecificGenerationHandle with WriteAgentSpecificFiles with AgentFormatBundleVariables] = { + CFEngineAgentSpecificGeneration :: DscAgentSpecificGeneration :: Nil + } + + override def write(cfg: AgentNodeWritableConfiguration): Box[List[AgentSpecificFile]] = { + (sequence(pipeline) { handler => + if(handler.handle(cfg.agentType)) { + handler.write(cfg) + } else { + Full(Nil) + } + }).map( _.flatten.toList) + } + + import BuildBundleSequence.{InputFile, TechniqueBundles, BundleSequenceVariables} + def getBundleVariables( + agentType : AgentType + , systemInputs: List[InputFile] + , sytemBundles: List[TechniqueBundles] + , userInputs : List[InputFile] + , userBundles : List[TechniqueBundles] + ) : Box[BundleSequenceVariables] = { + //we only choose the first matching agent for that + pipeline.find(handler => handler.handle(agentType)) match { + case None => Failure(s"We were unable to find how to create directive sequences for Agent type ${agentType.toString()}. " + + "Perhaps you are missing the corresponding plugin. If not, please report a bug") + case Some(h) => Full(h.getBundleVariables(systemInputs, sytemBundles, userInputs, userBundles)) + } + } + +} + + +trait AgentSpecificGeneration extends AgentSpecificGenerationHandle with AgentFormatBundleVariables with WriteAgentSpecificFiles + +object CFEngineAgentSpecificGeneration extends AgentSpecificGeneration { + val GENEREATED_CSV_FILENAME = "rudder_expected_reports.csv" + + + override def handle(agentType: AgentType): Boolean = agentType == AgentType.CfeCommunity || agentType == AgentType.CfeEnterprise + + override def write(cfg: AgentNodeWritableConfiguration): Box[List[AgentSpecificFile]] = { + writeExpectedReportsCsv(cfg.paths, cfg.expectedReportsCsv, GENEREATED_CSV_FILENAME) + } + + import BuildBundleSequence.{InputFile, TechniqueBundles, BundleSequenceVariables} + override def getBundleVariables( + systemInputs: List[InputFile] + , sytemBundles: List[TechniqueBundles] + , userInputs : List[InputFile] + , userBundles : List[TechniqueBundles] + ) : BundleSequenceVariables = CfengineBundleVariables.getBundleVariables(systemInputs, sytemBundles, userInputs, userBundles) + + + private[this] def writeExpectedReportsCsv(paths: NodePromisesPaths, csv: ExpectedReportsCsv, csvFilename: String): Box[List[AgentSpecificFile]] = { + val path = new File(paths.newFolder, csvFilename) + for { + _ <- tryo { FileUtils.writeStringToFile(path, csv.lines.mkString("\n"), Codec.UTF8.charSet) } ?~! + s"Can not write the expected reports CSV file at path '${path.getAbsolutePath}'" + } yield { + AgentSpecificFile(path.getAbsolutePath) :: Nil + } + } +} + +/* + * This will go in the plugin, and will be contributed somehow at config time. + */ +object DscAgentSpecificGeneration extends AgentSpecificGeneration { + + override def handle(agentType: AgentType): Boolean = agentType == AgentType.Dsc + + override def write(cfg: AgentNodeWritableConfiguration): Box[List[AgentSpecificFile]] = { + writeSystemVarJson(cfg.paths) + } + + import BuildBundleSequence.{InputFile, TechniqueBundles, BundleSequenceVariables} + override def getBundleVariables( + systemInputs: List[InputFile] + , sytemBundles: List[TechniqueBundles] + , userInputs : List[InputFile] + , userBundles : List[TechniqueBundles] + ) : BundleSequenceVariables = DscBundleVariables.getBundleVariables(systemInputs, sytemBundles, userInputs, userBundles) + + + // just write an empty file for now + private[this] def writeSystemVarJson(paths: NodePromisesPaths) = { + val path = new File(paths.newFolder, "rudder.json") + for { + _ <- tryo { FileUtils.writeStringToFile(path, """{ "comment":"for now, an empty file" }""" + "\n", Codec.UTF8.charSet) } ?~! + s"Can not write json parameter file at path '${path.getAbsolutePath}'" + } yield { + AgentSpecificFile(path.getAbsolutePath) :: Nil + } + } + +} diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/BuildBundleSequence.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/BuildBundleSequence.scala index c2c081c43c..9d7e783721 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/BuildBundleSequence.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/BuildBundleSequence.scala @@ -37,71 +37,19 @@ package com.normation.rudder.services.policies.write -import scala.annotation.migration -import scala.io.Codec - -import com.normation.cfclerk.domain.Bundle -import com.normation.cfclerk.domain.PARAMETER_VARIABLE -import com.normation.cfclerk.domain.SectionVariableSpec -import com.normation.cfclerk.domain.SystemVariable -import com.normation.cfclerk.domain.SystemVariableSpec +import com.normation.cfclerk.domain.BundleName import com.normation.cfclerk.domain.Technique -import com.normation.cfclerk.domain.TechniqueId -import com.normation.cfclerk.domain.TechniqueResourceId -import com.normation.cfclerk.domain.TrackerVariable -import com.normation.cfclerk.domain.TrackerVariableSpec -import com.normation.cfclerk.domain.Variable -import com.normation.cfclerk.exceptions.VariableException -import com.normation.cfclerk.services.SystemVariableSpecService -import com.normation.cfclerk.services.TechniqueRepository import com.normation.inventory.domain.NodeId -import com.normation.rudder.domain.reports.NodeConfigId +import com.normation.rudder.domain.policies.GlobalPolicyMode +import com.normation.rudder.domain.policies.PolicyMode import com.normation.rudder.services.policies.BundleOrder -import com.normation.rudder.services.policies.nodeconfig.NodeConfiguration -import com.normation.templates.STVariable import com.normation.utils.Control._ -import scala.language.implicitConversions - -import org.joda.time.DateTime - import net.liftweb.common._ -import com.normation.rudder.domain.policies.PolicyMode -import com.normation.rudder.domain.policies.GlobalPolicyMode +import com.normation.cfclerk.domain.SystemVariable +import com.normation.cfclerk.services.SystemVariableSpecService +import com.normation.inventory.domain.AgentType + -/* - * A data structure that holds the different values, with the different format, - * for each of the bundle related elements. - * - * Each value is just a correctly formatted string, which means for the template - * will never get the actual list of elements. This is a conscious decision that allows: - * - a simple type (String), available in any templating engine - * - the work is all done here, else most likelly for each evolution, it will have to - * be done here and in the template. As for now, system technique are separated - * from Rudder code repository, this is extremlly inifficient. - * - a consistant formatting (vertical align is impossible with string template) - */ -final case class BundleSequenceVariables( - // The first two variables are not used - // since 3.2, kept for migration compat from 3.1 - - // A list of unique file name to include. - inputlist : String - // the bundle list. - , bundlelist: String - - // The next four variable are used since 3.2 - - // the list of system inputs promise file - , systemDirectivesInputs : String - // the list of formated "usebundle" methods - // for system techniques - , systemDirectivesUsebundle: String - // the list of user inputs promise file - , directivesInputs : String - // the list of formated "usebundle" methods - // for user techniques - , directivesUsebundle: String -) /** * This file groups together everything related to building the bundle sequence and @@ -112,19 +60,63 @@ final case class BundleSequenceVariables( * - sort them accordingly to defined rules, * - add utility bundle when needed, like ncf logging bundle and dry-run mode */ -final object BuildBundleSequence extends Loggable { +object BuildBundleSequence { /* - * A cfengine input file to include + * A data structure that holds the different values, with the different format, + * for each of the bundle related elements. + * + * Each value is just a correctly formatted string, which means for the template + * will never get the actual list of elements. This is a conscious decision that allows: + * - a simple type (String), available in any templating engine + * - the work is all done here, else most likelly for each evolution, it will have to + * be done here and in the template. As for now, system technique are separated + * from Rudder code repository, this is extremlly inifficient. + * - a consistant formatting (vertical align is impossible with string template) + * + * We are returning List for more flexibility, knowing that: + * - in StringTemplate, Nil and List("") are not the same thing + * (you certainly want the latter) + * - for the CFEngine case at least, we are returning ONLY ONE + * string with the formatted list of inputs/bundles. + * We could return one input by line. + * */ - final case class Input(path: String, isSystem: Boolean) + final case class BundleSequenceVariables( + // the list of system inputs promise file + systemDirectivesInputFiles: List[String] + // the list of formated "usebundle" methods + // for system techniques + , systemDirectivesUsebundle : List[String] + // the list of user inputs promise file + , directivesInputFiles : List[String] + // the list of formated "usebundle" methods + // for user techniques + , directivesUsebundle : List[String] + ) + + /* + * An input file to include as a dependency + * (at least in cfengine) + */ + final case class InputFile(path: String, isSystem: Boolean) + + // ad-hoc data structure to denote a directive name + // (actually, the directive applied in to rule), + // or in CFEngine name a "promiser" + final case class Directive(value: String) + + // a Bundle is a BundleName and a Rudder Id that will + // be used to identify reports for that bundle + final case class Bundle(id: ReportId, name: BundleName) + + // The rudder id is directiveid@@ruleid@@serial + final case class ReportId(value: String) - //ad-hoc data structure to denote a promiser name - final case class Promiser(value: String) /* * A to-be-written list of bundle related to a unique - * technique. They all have the same promiser, derived + * technique. They all have the same Policy name, derived * from the technique/directive name. * * A list of bundle can be preceded and followed by some set-up bundles. @@ -136,7 +128,7 @@ final object BuildBundleSequence extends Loggable { * of bundle name, correctly sorted. */ final case class TechniqueBundles( - promiser : Promiser + promiser : Directive , pre : List[Bundle] , main : List[Bundle] // pre- and post-bundle sequence are simple bundle @@ -149,6 +141,10 @@ final object BuildBundleSequence extends Loggable { ) { def bundleSequence = pre ::: main ::: post } +} + +class BuildBundleSequence(systemVariableSpecService: SystemVariableSpecService) extends Loggable { + import BuildBundleSequence._ /* * The main entry point of the object: for each variable related to @@ -159,10 +155,11 @@ final object BuildBundleSequence extends Loggable { */ def prepareBundleVars( nodeId : NodeId + , agentType : AgentType , nodePolicyMode : Option[PolicyMode] , globalPolicyMode: GlobalPolicyMode , container : Cf3PolicyDraftContainer - ): Box[BundleSequenceVariables] = { + ): Box[List[SystemVariable]] = { logger.trace(s"Preparing bundle list and input list for node : ${nodeId.value}") @@ -180,33 +177,42 @@ final object BuildBundleSequence extends Loggable { // Then builds bundles and inputs: - // - build techniques bundles from the sorted list of techniques - val techniquesBundles = sortedTechniques.toList.map(buildTechniqueBundles(nodeId)).removeEmptyBundle.addNcfReporting - // - build list of inputs file to include: all the outPath of templates that should be "included". // (returns the pair of (outpath, isSystem) ) - val inputs: Seq[Input] = sortedTechniques.flatMap {case (technique, _, _) => - technique.templates.collect { case template if(template.included) => Input(template.outPath, technique.isSystem) } ++ - technique.files.collect { case file if(file.included) => Input(file.outPath, technique.isSystem) } + val inputs: List[InputFile] = sortedTechniques.flatMap {case (technique, _, _, _) => + technique.agentConfigs.find( _.agentType == agentType) match { + case None => Nil + case Some(cfg) => + cfg.templates.collect { case template if(template.included) => InputFile(template.outPath, technique.isSystem) } ++ + cfg.files.collect { case file if(file.included) => InputFile(file.outPath, technique.isSystem) } + } + }.toList + + //split system and user inputs + val (systemInputFiles, userInputFiles) = inputs.partition( _.isSystem ) + + // get the output string for each bundle variables, agent dependant + for { + // - build techniques bundles from the sorted list of techniques + techniquesBundles <- sequence(sortedTechniques)(buildTechniqueBundles(nodeId, agentType)) + //split system and user directive (technique) + (systemBundle, userBundle) = techniquesBundles.toList.removeEmptyBundle.partition( _.isSystem ) + bundleVars <- WriteAllAgentSpecificFiles.getBundleVariables(agentType, systemInputFiles, systemBundle, userInputFiles, userBundle) + } yield { + + // map to correct variables + List( + //this one is CFengine specific and kept for historical reason + SystemVariable(systemVariableSpecService.get("INPUTLIST") , CfengineBundleVariables.formatBundleFileInputFiles(inputs.map(_.path))) + //this one is CFengine specific and kept for historical reason + , SystemVariable(systemVariableSpecService.get("BUNDLELIST"), techniquesBundles.flatMap( _.bundleSequence.map(_.name)).mkString(", ", ", ", "") :: Nil) + , SystemVariable(systemVariableSpecService.get("RUDDER_SYSTEM_DIRECTIVES_INPUTS") , bundleVars.systemDirectivesInputFiles) + , SystemVariable(systemVariableSpecService.get("RUDDER_SYSTEM_DIRECTIVES_SEQUENCE"), bundleVars.systemDirectivesUsebundle) + , SystemVariable(systemVariableSpecService.get("RUDDER_DIRECTIVES_INPUTS") , bundleVars.directivesInputFiles) + , SystemVariable(systemVariableSpecService.get("RUDDER_DIRECTIVES_SEQUENCE") , bundleVars.directivesUsebundle) + ) } - - //split system and user directive (technique) - val (systemInputs, userInputs) = inputs.partition( _.isSystem ) - val (systemBundle, userBundle) = techniquesBundles.partition( _.isSystem ) - - //only user bundle may be set on PolicyMode = Verify - val userBundleWithDryMode = userBundle.addDryRunManagement - - // All done, return all the correctly formatted system variables - Full(BundleSequenceVariables( - inputlist = formatBundleFileInputs(inputs.map(_.path)) - , bundlelist = techniquesBundles.flatMap( _.bundleSequence.map(_.name)).mkString(", ", ", ", "") - , systemDirectivesInputs = formatBundleFileInputs(systemInputs.map(_.path)) - , systemDirectivesUsebundle = formatMethodsUsebundle(systemBundle) - , directivesInputs = formatBundleFileInputs(userInputs.map(_.path)) - , directivesUsebundle = formatMethodsUsebundle(userBundleWithDryMode) - )) } } @@ -215,38 +221,155 @@ final object BuildBundleSequence extends Loggable { //////////////////////////////////////////// /* - * For each techniques, build: - * - the promiser - * - the list of bundle included, + * Some Techniques don't have any bundle (at least common). + * We don't want to include these technique in the bundle sequence, + * obviously */ - def buildTechniqueBundles(nodeId: NodeId)(t3: (Technique, List[BundleOrder], PolicyMode)): TechniqueBundles = { + implicit final class NoBundleTechnique(bundles: List[TechniqueBundles]) { + def removeEmptyBundle: List[TechniqueBundles] = bundles.filterNot(_.main.isEmpty) + } + + + /* + * For each techniques: + * - check the node AgentType and fails if the technique is not compatible with it + * - build the name + * - build the list of bundle included, + * + * The List[BundleOrder] is actually List(ruleName, directiveName) for the chose couple for that technique. + * The ReportId is the same for all bundles, because we don't have a better granularity for now + * (and it is also why we get it from sortTechniques, which is kind of strange :) + * + */ + def buildTechniqueBundles(nodeId: NodeId, agentType: AgentType)(t3: (Technique, ReportId, List[BundleOrder], PolicyMode)): Box[TechniqueBundles] = { // naming things to make them clear val technique = t3._1 - val promiser = Promiser(t3._2.map(_.value).mkString("/")) - val policyMode = t3._3 - - // We need to remove zero-length bundle name from the bundlesequence (like, if there is no ncf bundles to call) - // to avoid having two successives commas in the bundlesequence - val techniqueBundles = technique.bundlesequence.flatMap { bundle => - if(bundle.name.trim.size > 0) { - Some(bundle) - } else { - logger.warn(s"Technique '${technique.id}' used in node '${nodeId.value}' contains some bundle with empty name, which is forbidden and so they are ignored in the final bundle sequence") - None - } - }.toList - - TechniqueBundles(promiser, Nil, techniqueBundles, Nil, technique.isSystem, technique.providesExpectedReports, policyMode) + val reportId = t3._2 + val name = Directive(t3._3.map(_.value).mkString("/")) + val policyMode = t3._4 + + // and for now, all bundle get the same reportKey + technique.agentConfigs.find(_.agentType == agentType) match { + case Some(cfg) => + val techniqueBundles = cfg.bundlesequence.map { bundleName => + if(bundleName.value.trim.size > 0) { + List(Bundle(reportId, bundleName)) + } else { + logger.warn(s"Technique '${technique.id}' used in node '${nodeId.value}' contains some bundle with empty name, which is forbidden and so they are ignored in the final bundle sequence") + Nil + } + }.flatten.toList + Full(TechniqueBundles(name, Nil, techniqueBundles, Nil, technique.isSystem, technique.providesExpectedReports, policyMode)) + case None => + Failure(s"Node with id '${nodeId.value}' is configured with agent type='${agentType.toString}' but technique '${technique.id.toString}' is not compatible with that agent type.") + } } /* - * Some Techniques don't have any bundle (at least common). - * We don't want to include these technique in the bundle sequence, - * obviously + * Sort the techniques according to the order of the associated BundleOrder of Cf3PolicyDraft. + * Sort at best: sort rule then directives, and take techniques on that order, only one time. + * + * CAREFUL: this method only take care of sorting based on "BundleOrder", other sorting (like + * "system must go first") are not taken into account here ! + * + * Actually, sortTechnique does more because it is the point where we look if a technique is + * used in several rules/directives and where we choose what rule/directive will be used, + * so we are also looking for: + * - the Audit Mode consistancy (and return it if consistant), + * - the ReportId of the chosen couple. + * */ - implicit final class NoBundleTechnique(bundles: List[TechniqueBundles]) { - def removeEmptyBundle: List[TechniqueBundles] = bundles.filterNot(_.main.isEmpty) + def sortTechniques( + nodeId : NodeId + , nodePolicyMode : Option[PolicyMode] + , globalPolicyMode: GlobalPolicyMode + , container : Cf3PolicyDraftContainer + ): Box[Seq[(Technique, ReportId, List[BundleOrder], PolicyMode)]] = { + + val techniques = container.getTechniques().values.toList + + def compareBundleOrder(a: Cf3PolicyDraft, b: Cf3PolicyDraft): Boolean = { + BundleOrder.compareList(List(a.ruleOrder, a.directiveOrder), List(b.ruleOrder, b.directiveOrder)) <= 0 + } + + val drafts = container.cf3PolicyDrafts.values.toList + + for { + //for each technique, get it's best order from draft (if several directive use it) + techBundle <- bestEffort(techniques) { t => + + val tDrafts = drafts.filter { _.technique.id == t.id }.sortWith( compareBundleOrder ) + + tDrafts match { + case Nil => //that should not happen because we only chose techniques with draft, but still + Failure(s"Error with Technique ${t.name} during bundle sequence generation, we found a techniques without any bundle" + + s" for node '${nodeId.value}' but it should not happen. " + + "This is likely a bug, please report it to https://rudder-project.org/issues") + case draft :: _ => + // the bundle full order in the couple (rule name, directive name) + val bundleOrder = List(draft.ruleOrder, draft.directiveOrder) + // the rudderId is homogeneous for the whole technique + val rudderId = ReportId(s"${draft.id.value}@@${draft.serial}") + + // until we have a directive-level granularity for policy mode, we have to define a technique-level policy mode + // if directives don't have a consistant policy mode, we fail (and report it) + (PolicyMode.computeMode(globalPolicyMode, nodePolicyMode, tDrafts.map(_.policyMode)).map { policyMode => + (t, rudderId, bundleOrder, policyMode) + }) match { + case Full(x) => Full(x) + case eb: EmptyBox => + val msg = eb match { + case Empty => "" + case Failure(m, _, _) => ": " + m + } + Failure(s"Error with Technique ${t.name} and [Rule ID // Directives ID: Policy Mode] ${tDrafts.map { x => + s"[${x.id.ruleId.value} // ${x.id.directiveId.value}: ${x.policyMode.map(_.name).getOrElse("inherited")}]" + }.mkString("; ") } ${msg}") + } + } + } + } yield { + + //now that we have the (rule/directive) for each technique, sort techniques + val ordered = techBundle.sortWith { case ((_, _, o1, _), (_, _, o2, _)) => BundleOrder.compareList(o1, o2) <= 0 } + + //some debug info to understand what order was used for each node: + if(logger.isDebugEnabled) { + val sorted = ordered.map(p => s"${p._1.name}: [${p._3.map(_.value).mkString(" | ")}]").mkString("[","][", "]") + logger.debug(s"Sorted Technique (and their Rules and Directives used to sort): ${sorted}") + } + + ordered + } } +} + +////////////////////////////////////////////////////////////////////// +////////// Agent specific implementation to format outputs ////////// +////////////////////////////////////////////////////////////////////// + + + +object CfengineBundleVariables extends AgentFormatBundleVariables { + import BuildBundleSequence._ + + override def getBundleVariables( + systemInputs: List[InputFile] + , sytemBundles: List[TechniqueBundles] + , userInputs : List[InputFile] + , userBundles : List[TechniqueBundles] + ) : BundleSequenceVariables = { + + + BundleSequenceVariables( + formatBundleFileInputFiles(systemInputs.map(_.path)) + , formatMethodsUsebundle(sytemBundles.addNcfReporting) + , formatBundleFileInputFiles(userInputs.map(_.path)) + //only user bundle may be set on PolicyMode = Verify + , formatMethodsUsebundle(userBundles.addNcfReporting.addDryRunManagement) + ) + } + /* * The logic that is in charge to add ncf reporting information @@ -261,7 +384,8 @@ final object BuildBundleSequence extends Loggable { if(tb.providesExpectedReports) { val newMain = tb.main match { case Nil => Nil - case firstBundle :: tail => (Bundle(s"""current_technique_report_info("${firstBundle.name}")""") + // in that case, we are using the same ReportId for report and bundle + case firstBundle :: tail => (Bundle(firstBundle.id, BundleName(s"""current_technique_report_info("${firstBundle.name.value}")""")) :: firstBundle :: tail) } tb.copy(main = newMain) @@ -292,15 +416,23 @@ final object BuildBundleSequence extends Loggable { } //before each technique, set the correct mode - private[this] val audit = Bundle("""set_dry_run_mode("true")""") - private[this] val enforce = Bundle("""set_dry_run_mode("false")""") - private[this] val cleanup = TechniqueBundles(Promiser("remove_dry_run_mode"), Nil, enforce :: Nil, Nil, false, false, PolicyMode.Enforce) + private[this] val audit = Bundle(ReportId("internal"), BundleName("""set_dry_run_mode("true")""")) + private[this] val enforce = Bundle(ReportId("internal"), BundleName("""set_dry_run_mode("false")""")) + private[this] val cleanup = TechniqueBundles(Directive("remove_dry_run_mode"), Nil, enforce :: Nil, Nil, false, false, PolicyMode.Enforce) } + /* * Method for formating list of "promiser usebundle => bundlename;" + * + * For the CFengine agent, we are waiting ONE string of the fully + * formatted result, ie. something like: """ + * "Rule1/directive one" usebundle => fetchFusionTools; + * "Rule1/some other directive with a long name" usebundle => doInventory; + * "An other rule/its directive" usebundle => virtualMachines; + * """ */ - def formatMethodsUsebundle(bundleSeq: Seq[TechniqueBundles]): String = { + def formatMethodsUsebundle(bundleSeq: Seq[TechniqueBundles]): List[String] = { //the promiser value (may) comes from user input, so we need to escape //also, get the list of bundle for each promiser. //and we don't need isSystem anymore @@ -310,85 +442,74 @@ final object BuildBundleSequence extends Loggable { //number in all Rudder ! val alignWidth = if(escapedSeq.size <= 0) 0 else escapedSeq.map(_._1.size).max - escapedSeq.flatMap { case (promiser, bundles) => + (escapedSeq.flatMap { case (promiser, bundles) => bundles.map { bundle => - s""""${promiser}"${ " " * Math.max(0, alignWidth - promiser.size) } usebundle => ${bundle.name};""" + s""""${promiser}"${ " " * Math.max(0, alignWidth - promiser.size) } usebundle => ${bundle.name.value};""" } - }.mkString( "\n") + }.mkString( "\n")) :: Nil } /* * utilitary method for formating an input list + * For the CFengine agent, we are waiting ONE string of the fully + * formatted result, ie. something like: """ + * "common/1.0/update.cf", + * "rudder-directives.cf", + * "rudder-system-directives.cf", + * "common/1.0/rudder-parameters.cf", + * """ */ - def formatBundleFileInputs(x: Seq[String]) = { + def formatBundleFileInputFiles(x: Seq[String]): List[String] = { val inputs = x.distinct if (inputs.isEmpty) { - "" + List("") //we must have one empty parameter to have the awaited behavior with string template } else { - inputs.mkString("\"", s"""",\n"""", s"""",""") + List(inputs.mkString("\"", s"""",\n"""", s"""",""")) } } +} + +object DscBundleVariables extends AgentFormatBundleVariables { + import BuildBundleSequence._ + + override def getBundleVariables( + systemInputs: List[InputFile] + , sytemBundles: List[TechniqueBundles] + , userInputs : List[InputFile] + , userBundles : List[TechniqueBundles] + ) : BundleSequenceVariables = { + BundleSequenceVariables( + // no use of input files in DSC + Nil + // no specific pre/post bundles added for DSC + , formatMethodsUsebundle(sytemBundles) + // no use of input files in DSC + , Nil + // no specific pre/post bundles added for DSC + , formatMethodsUsebundle(userBundles) + ) + } /* - * Sort the techniques according to the order of the associated BundleOrder of Cf3PolicyDraft. - * Sort at best: sort rule then directives, and take techniques on that order, only one time - * Sort system directive first. - * - * CAREFUL: this method only take care of sorting based on "BundleOrder", other sorting (like - * "system must go first") are not taken into account here ! + * Method for formating list of "TechniqueName -ReportId XXX -TechniqueName "the name" -AuditMode $true " */ - def sortTechniques( - nodeId : NodeId - , nodePolicyMode : Option[PolicyMode] - , globalPolicyMode: GlobalPolicyMode - , container : Cf3PolicyDraftContainer - ): Box[Seq[(Technique, List[BundleOrder], PolicyMode)]] = { - - val techniques = container.getTechniques().values.toList - - def compareBundleOrder(a: Cf3PolicyDraft, b: Cf3PolicyDraft): Boolean = { - BundleOrder.compareList(List(a.ruleOrder, a.directiveOrder), List(b.ruleOrder, b.directiveOrder)) <= 0 - } - - val drafts = container.cf3PolicyDrafts.values.toSeq - - for { - //for each technique, get it's best order from draft (if several directive use it) and return a pair (technique, List(order)) - triples <- bestEffort(techniques) { t => - - val tDrafts = drafts.filter { _.technique.id == t.id }.sortWith( compareBundleOrder ) - - //the order we want is the one with the lowest draft order, or the default one if no draft found (but that should not happen by construction) - val order = tDrafts.map( t => List(t.ruleOrder, t.directiveOrder)).headOption.getOrElse(List(BundleOrder.default)) - - // until we have a directive-level granularity for policy mode, we have to define a technique-level policy mode - // if directives don't have a consistant policy mode, we fail (and report it) - (PolicyMode.computeMode(globalPolicyMode, nodePolicyMode, tDrafts.map(_.policyMode)).map { policyMode => - (t, order, policyMode) - }) match { - case Full(x) => Full(x) - case eb: EmptyBox => - val msg = eb match { - case Empty => "" - case Failure(m, _, _) => ": " + m - } - Failure(s"Error with Technique ${t.name} and [Rule ID // Directives ID: Policy Mode] ${tDrafts.map { x => - s"[${x.id.ruleId.value} // ${x.id.directiveId.value}: ${x.policyMode.map(_.name).getOrElse("inherited")}]" - }.mkString("; ") } ${msg}") - } - } - } yield { - - //now just sort the pair by order and keep only techniques - val ordered = triples.sortWith { case ((_, o1, _), (_, o2, _)) => BundleOrder.compareList(o1, o2) <= 0 } + def formatMethodsUsebundle(bundleSeq: Seq[TechniqueBundles]): List[String] = { - //some debug info to understand what order was used for each node: - if(logger.isDebugEnabled) { - val sorted = ordered.map(p => s"${p._1.name}: [${p._2.map(_.value).mkString(" | ")}]").mkString("[","][", "]") - logger.debug(s"Sorted Technique (and their Rules and Directives used to sort): ${sorted}") + (bundleSeq.flatMap { technique => + val auditOnly = technique.policyMode match { + case PolicyMode.Audit => "true" + case _ => "false" } - ordered - } + technique.bundleSequence.map { bundle => + //we don't have exactly the same output for system and non system technique + if(technique.isSystem) { + s""" ${bundle.name.value} -ReportId ${bundle.id.value}""" + } else { + s""" ${bundle.name.value} -ReportId ${bundle.id.value} -TechniqueName "${technique.promiser.value}" -AuditOnly $$${auditOnly}""" + } + } + }.mkString( "\n")) :: Nil } + } diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PathComputer.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PathComputer.scala index fa4ad7afdf..ed2bc4ecbc 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PathComputer.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PathComputer.scala @@ -43,10 +43,8 @@ import com.normation.inventory.domain.NodeId import com.normation.inventory.domain.AgentType import net.liftweb.common.Loggable import com.normation.exceptions.BusinessException -import com.normation.rudder.services.policies.nodeconfig.NodeConfiguration import net.liftweb.common.Full import net.liftweb.common.Box -import com.normation.rudder.domain.Constants import org.apache.commons.io.FilenameUtils import com.normation.rudder.domain.nodes.NodeInfo import net.liftweb.common.Failure diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/Cf3PromisesFileWriterService.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala similarity index 92% rename from rudder-core/src/main/scala/com/normation/rudder/services/policies/write/Cf3PromisesFileWriterService.scala rename to rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala index a1f01398ef..3287adbb2d 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/Cf3PromisesFileWriterService.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PolicyWriterService.scala @@ -37,64 +37,52 @@ package com.normation.rudder.services.policies.write -import java.io.File -import java.io.IOException -import com.normation.cfclerk.domain.Technique import com.normation.cfclerk.domain.TechniqueFile import com.normation.cfclerk.domain.TechniqueId import com.normation.cfclerk.domain.TechniqueResourceId import com.normation.cfclerk.domain.TechniqueTemplate -import com.normation.cfclerk.exceptions.VariableException import com.normation.cfclerk.services.TechniqueRepository import com.normation.inventory.domain.AgentType -import com.normation.inventory.domain.AgentType.CfeCommunity import com.normation.inventory.domain.AgentType.CfeEnterprise import com.normation.inventory.domain.NodeId -import com.normation.inventory.domain.NodeId -import com.normation.rudder.domain.reports.NodeConfigId +import com.normation.rudder.domain.Constants +import com.normation.rudder.domain.licenses.CfeEnterpriseLicense +import com.normation.rudder.domain.nodes.NodeProperty +import com.normation.rudder.domain.policies.GlobalPolicyMode import com.normation.rudder.domain.reports.NodeConfigId -import com.normation.rudder.repository.LicenseRepository -import com.normation.rudder.services.policies.nodeconfig.NodeConfiguration +import com.normation.rudder.hooks.HookEnvPairs +import com.normation.rudder.hooks.HooksLogger +import com.normation.rudder.hooks.RunHooks import com.normation.rudder.services.policies.nodeconfig.NodeConfiguration import com.normation.rudder.services.policies.nodeconfig.NodeConfigurationLogger +import com.normation.templates.FillTemplatesService +import com.normation.templates.STVariable import com.normation.utils.Control._ -import org.antlr.stringtemplate.StringTemplate +import java.io.File +import java.io.IOException +import monix.eval.Task +import monix.eval.TaskSemaphore +import monix.execution.ExecutionModel +import monix.execution.Scheduler +import net.liftweb.common._ +import net.liftweb.json.JsonAST +import net.liftweb.json.JsonAST.JValue +import net.liftweb.util.Helpers.tryo import org.apache.commons.io.FileUtils import org.apache.commons.io.FilenameUtils import org.apache.commons.io.IOUtils import org.joda.time.DateTime -import org.joda.time.LocalDate -import org.joda.time.LocalTime -import net.liftweb.common._ -import net.liftweb.util.Helpers.tryo -import com.normation.rudder.domain.licenses.CfeEnterpriseLicense +import scala.concurrent.Await import scala.io.Codec -import com.normation.templates.FillTemplatesService -import com.normation.templates.STVariable -import com.normation.rudder.domain.nodes.NodeProperty -import net.liftweb.json.JsonAST.JValue -import scala.util.Try +import scala.util.{ Failure => FailTry } import scala.util.Success -import scala.util.{Failure => FailTry} -import com.normation.rudder.domain.Constants -import net.liftweb.json.JsonAST -import net.liftweb.json.Printer -import com.normation.rudder.domain.policies.GlobalPolicyMode -import scala.language.postfixOps -import com.normation.rudder.hooks.RunHooks -import com.normation.rudder.hooks.HookEnvPairs -import com.normation.rudder.hooks.HooksLogger -import monix.execution.Scheduler -import monix.eval.TaskSemaphore -import monix.eval.Task -import scala.concurrent.Await -import monix.execution.ExecutionModel +import scala.util.Try /** * Write promises for the set of nodes, with the given configs. * Requires access to external templates files. */ -trait Cf3PromisesFileWriterService { +trait PolicyWriterService { /** * Write templates for node configuration that changed since the last write. @@ -119,10 +107,9 @@ class Cf3PromisesFileWriterServiceImpl( , fillTemplates : FillTemplatesService , HOOKS_D : String , HOOKS_IGNORE_SUFFIXES: List[String] -) extends Cf3PromisesFileWriterService with Loggable { +) extends PolicyWriterService with Loggable { val TAG_OF_RUDDER_ID = "@@RUDDER_ID@@" - val GENEREATED_CSV_FILENAME = "rudder_expected_reports.csv" val newPostfix = ".new" val backupPostfix = ".bkp" @@ -130,8 +117,8 @@ class Cf3PromisesFileWriterServiceImpl( private[this] def writeNodePropertiesFile (agentNodeConfig: AgentNodeConfiguration) = { def generateNodePropertiesJson(properties : Seq[NodeProperty]): JValue = { - import net.liftweb.json.JsonDSL._ import com.normation.rudder.domain.nodes.JsonSerialisation._ + import net.liftweb.json.JsonDSL._ ( "properties" -> properties.toDataJson()) } @@ -264,7 +251,8 @@ class Cf3PromisesFileWriterServiceImpl( }) match { case s if s.charAt(0) == 'x' => (Runtime.getRuntime.availableProcessors * s.substring(1).toDouble).ceil.toInt case other => other.toInt - }}) + } + }) implicit val scheduler = Scheduler.io(executionModel = ExecutionModel.AlwaysAsyncExecution) //interpret HookReturnCode as a Box @@ -274,23 +262,39 @@ class Cf3PromisesFileWriterServiceImpl( configAndPaths <- calculatePathsForNodeConfigurations(interestingNodeConfigs, rootNodeId, allNodeConfigs, newPostfix, backupPostfix) pathsInfo = configAndPaths.map { _.paths } templates <- readTemplateFromFileSystem(techniqueIds) + + ////////// + // nothing agent specific before that + ////////// + + preparedPromises <- parrallelSequence(configAndPaths) { case agentNodeConfig => - val nodeConfigId = versions(agentNodeConfig.config.nodeInfo.id) - prepareTemplate.prepareTemplateForAgentNodeConfiguration(agentNodeConfig, nodeConfigId, rootNodeId, templates, allNodeConfigs, TAG_OF_RUDDER_ID, globalPolicyMode) - } + val nodeConfigId = versions(agentNodeConfig.config.nodeInfo.id) + prepareTemplate.prepareTemplateForAgentNodeConfiguration(agentNodeConfig, nodeConfigId, rootNodeId, templates, allNodeConfigs, TAG_OF_RUDDER_ID, globalPolicyMode) + } promiseWritten <- parrallelSequence(preparedPromises) { prepared => for { _ <- writePromises(prepared.paths, prepared.preparedTechniques) - _ <- writeExpectedReportsCsv(prepared.paths, prepared.expectedReportsCsv, GENEREATED_CSV_FILENAME) + _ <- WriteAllAgentSpecificFiles.write(prepared) } yield { "OK" } } - propertiesWritten <- parrallelSequence(configAndPaths) { case agentNodeConfig => + + ////////// + // nothing agent specific after that + ////////// + + propertiesWritten <- parrallelSequence(configAndPaths) { case agentNodeConfig => writeNodePropertiesFile(agentNodeConfig) ?~! s"An error occured while writing property file for Node ${agentNodeConfig.config.nodeInfo.hostname} (id: ${agentNodeConfig.config.nodeInfo.id.value}" } + licensesCopied <- copyLicenses(configAndPaths, allLicenses) + + /// perhaps that should be a post-hook somehow ? + // and perhaps we should have an AgentSpecific global pre/post write + nodePreMvHooks <- RunHooks.getHooks(HOOKS_D + "/policy-generation-node-ready", HOOKS_IGNORE_SUFFIXES) preMvHooks <- parrallelSequence(configAndPaths) { agentNodeConfig => val timeHooks = System.currentTimeMillis @@ -379,21 +383,29 @@ class Cf3PromisesFileWriterServiceImpl( } } + /* + * We are returning a map where keys are (TechniqueResourceId, AgentType) because + * for a given resource IDs, you can have different out path for different agent. + */ private[this] def readTemplateFromFileSystem( techniqueIds: Set[TechniqueId] - )(implicit scheduler: Scheduler, semaphore: TaskSemaphore): Box[Map[TechniqueResourceId, TechniqueTemplateCopyInfo]] = { + )(implicit scheduler: Scheduler, semaphore: TaskSemaphore): Box[Map[(TechniqueResourceId, AgentType), TechniqueTemplateCopyInfo]] = { //list of (template id, template out path) val templatesToRead = for { technique <- techniqueRepository.getByIds(techniqueIds.toSeq) - template <- technique.templates + template <- technique.agentConfigs.flatMap(cfg => cfg.templates.map(t => (t.id, cfg.agentType, t.outPath))) } yield { - (template.id, template.outPath) + template } val now = System.currentTimeMillis() - val res = (parrallelSequence(templatesToRead) { case (templateId, templateOutPath) => + /* + * NOTE : this is inefficient and store in a lot of multiple time the same content + * if only the outpath change for two differents agent type. + */ + val res = (parrallelSequence(templatesToRead) { case (templateId, agentType, templateOutPath) => for { copyInfo <- techniqueRepository.getTemplateContent(templateId) { optInputStream => optInputStream match { @@ -407,7 +419,7 @@ class Cf3PromisesFileWriterServiceImpl( } } } yield { - (copyInfo.id, copyInfo) + ((copyInfo.id, agentType), copyInfo) } }).map( _.toMap) @@ -415,18 +427,8 @@ class Cf3PromisesFileWriterServiceImpl( res } - private[this] def writeExpectedReportsCsv(paths: NodePromisesPaths, csv: ExpectedReportsCsv, csvFilename: String): Box[String] = { - val path = new File(paths.newFolder, csvFilename) - for { - _ <- tryo { FileUtils.writeStringToFile(path, csv.lines.mkString("\n"), Codec.UTF8.charSet) } ?~! - s"Can not write the expected reports CSV file at path '${path.getAbsolutePath}'" - } yield { - path.getAbsolutePath - } - } - private[this] def writePromises( - paths : NodePromisesPaths + paths : NodePromisesPaths , preparedTechniques: Seq[PreparedTechnique] ) : Box[NodePromisesPaths] = { // write the promises of the current machine and set correct permission @@ -448,6 +450,7 @@ class Cf3PromisesFileWriterServiceImpl( } } + /** * For agent needing it, copy licences to the correct path */ diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariables.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariables.scala index fe093ca65e..ae80958c5d 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariables.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariables.scala @@ -37,13 +37,8 @@ package com.normation.rudder.services.policies.write -import scala.annotation.migration -import scala.io.Codec - -import com.normation.cfclerk.domain.Bundle import com.normation.cfclerk.domain.PARAMETER_VARIABLE import com.normation.cfclerk.domain.SectionVariableSpec -import com.normation.cfclerk.domain.SystemVariable import com.normation.cfclerk.domain.SystemVariableSpec import com.normation.cfclerk.domain.Technique import com.normation.cfclerk.domain.TechniqueId @@ -57,18 +52,17 @@ import com.normation.cfclerk.services.TechniqueRepository import com.normation.inventory.domain.AgentType.CfeCommunity import com.normation.inventory.domain.AgentType.CfeEnterprise import com.normation.inventory.domain.NodeId +import com.normation.rudder.domain.policies.GlobalPolicyMode +import com.normation.rudder.domain.policies.PolicyMode import com.normation.rudder.domain.reports.NodeConfigId -import com.normation.rudder.services.policies.BundleOrder import com.normation.rudder.services.policies.nodeconfig.NodeConfiguration import com.normation.templates.STVariable import com.normation.utils.Control._ -import scala.language.implicitConversions - -import org.joda.time.DateTime - import net.liftweb.common._ -import com.normation.rudder.domain.policies.GlobalPolicyMode -import com.normation.rudder.domain.policies.PolicyMode +import org.joda.time.DateTime +import scala.io.Codec +import com.normation.inventory.domain.AgentType +import com.normation.cfclerk.domain.TechniqueFile trait PrepareTemplateVariables { @@ -86,7 +80,7 @@ trait PrepareTemplateVariables { agentNodeConfig : AgentNodeConfiguration , nodeConfigVersion: NodeConfigId , rootNodeConfigId : NodeId - , templates : Map[TechniqueResourceId, TechniqueTemplateCopyInfo] + , templates : Map[(TechniqueResourceId, AgentType), TechniqueTemplateCopyInfo] , allNodeConfigs : Map[NodeId, NodeConfiguration] , rudderIdCsvTag : String , globalPolicyMode : GlobalPolicyMode @@ -101,13 +95,14 @@ trait PrepareTemplateVariables { class PrepareTemplateVariablesImpl( techniqueRepository : TechniqueRepository // only for getting reports file content , systemVariableSpecService: SystemVariableSpecService + , buildBundleSequence : BuildBundleSequence ) extends PrepareTemplateVariables with Loggable { override def prepareTemplateForAgentNodeConfiguration( agentNodeConfig : AgentNodeConfiguration , nodeConfigVersion: NodeConfigId , rootNodeConfigId : NodeId - , templates : Map[TechniqueResourceId, TechniqueTemplateCopyInfo] + , templates : Map[(TechniqueResourceId, AgentType), TechniqueTemplateCopyInfo] , allNodeConfigs : Map[NodeId, NodeConfiguration] , rudderIdCsvTag : String , globalPolicyMode : GlobalPolicyMode @@ -130,25 +125,33 @@ class PrepareTemplateVariablesImpl( ).map(x => (x.spec.name, x)).toMap for { - bundleVars <- prepareBundleVars(agentNodeConfig.config.nodeInfo.id, agentNodeConfig.config.nodeInfo.policyMode, globalPolicyMode, container) + bundleVars <- prepareBundleVars(agentNodeConfig.config.nodeInfo.id, agentNodeConfig.agentType, agentNodeConfig.config.nodeInfo.policyMode, globalPolicyMode, container) } yield { val allSystemVars = systemVariables.toMap ++ bundleVars val preparedTemplate = prepareTechniqueTemplate( - agentNodeConfig.config.nodeInfo.id, container, allSystemVars, templates, generationTimestamp + agentNodeConfig.config.nodeInfo.id, agentNodeConfig.agentType, container, allSystemVars, templates, generationTimestamp ) - logger.trace(s"${agentNodeConfig.config.nodeInfo.id.value}: creating lines for expected reports CSV files") - val csv = ExpectedReportsCsv(prepareReportingDataForMetaTechnique(container, rudderIdCsvTag)) - - AgentNodeWritableConfiguration(agentNodeConfig.paths, preparedTemplate.values.toSeq, csv) + // we only need to generate expected_reports.csv for CFEngine-like agent + val csv = ExpectedReportsCsv(agentNodeConfig.agentType match { + case AgentType.CfeCommunity | AgentType.CfeEnterprise => + logger.trace(s"${agentNodeConfig.config.nodeInfo.id.value}: creating lines for expected reports CSV files") + prepareReportingDataForMetaTechnique(container, rudderIdCsvTag) + case _ => // DSC_AGENT or other + logger.trace(s"${agentNodeConfig.config.nodeInfo.id.value}: not creating lines for expected reports CSV files - agent type '${agentNodeConfig.agentType.fullname}' does not need them") + Seq() + }) + + AgentNodeWritableConfiguration(agentNodeConfig.agentType, agentNodeConfig.paths, preparedTemplate.values.toSeq, csv) } } private[this] def prepareTechniqueTemplate( nodeId : NodeId // for log message + , agentType : AgentType , container : Cf3PolicyDraftContainer , extraSystemVariables: Map[String, Variable] - , allTemplates : Map[TechniqueResourceId, TechniqueTemplateCopyInfo] + , allTemplates : Map[(TechniqueResourceId, AgentType), TechniqueTemplateCopyInfo] , generationTimestamp : Long ) : Map[TechniqueId, PreparedTechnique] = { @@ -169,14 +172,19 @@ class PrepareTemplateVariablesImpl( val generationVariable = STVariable("GENERATIONTIMESTAMP", false, Seq(generationTimestamp), true) techniques.map {technique => - val techniqueTemplatesIds = technique.templatesMap.keySet + val techniqueTemplatesIds = technique.templatesIds // this is an optimisation to avoid re-redeading template each time, for each technique. We could // just, from a strict correctness point of view, just do a techniquerepos.getcontent for each technique.template here - val techniqueTemplates = allTemplates.filterKeys(k => techniqueTemplatesIds.contains(k)).values.toSet + val techniqueTemplates = allTemplates.filterKeys(k => techniqueTemplatesIds.contains(k._1) && k._2 == agentType).values.toSet val variables = variablesByTechnique(technique.id) :+ rudderParametersVariable :+ generationVariable + val files = (technique.agentConfigs.find( _.agentType == agentType).map( _.files) match { + case None => Set.empty[TechniqueFile] + case Some(x) => x.toSet[TechniqueFile] + }) + ( technique.id - , PreparedTechnique(techniqueTemplates, variables, technique.files.toSet) + , PreparedTechnique(techniqueTemplates, variables, files) ) }.toMap } @@ -262,21 +270,14 @@ class PrepareTemplateVariablesImpl( */ private[this] def prepareBundleVars( nodeId : NodeId + , agentType : AgentType , nodePolicyMode : Option[PolicyMode] , globalPolicyMode: GlobalPolicyMode , container : Cf3PolicyDraftContainer ) : Box[Map[String,Variable]] = { - BuildBundleSequence.prepareBundleVars(nodeId, nodePolicyMode, globalPolicyMode, container).map { bundleVars => - - List( - SystemVariable(systemVariableSpecService.get("INPUTLIST") , bundleVars.inputlist :: Nil) - , SystemVariable(systemVariableSpecService.get("BUNDLELIST"), bundleVars.bundlelist :: Nil) - , SystemVariable(systemVariableSpecService.get("RUDDER_SYSTEM_DIRECTIVES_INPUTS") , bundleVars.systemDirectivesInputs :: Nil) - , SystemVariable(systemVariableSpecService.get("RUDDER_SYSTEM_DIRECTIVES_SEQUENCE"), bundleVars.systemDirectivesUsebundle :: Nil) - , SystemVariable(systemVariableSpecService.get("RUDDER_DIRECTIVES_INPUTS") , bundleVars.directivesInputs :: Nil) - , SystemVariable(systemVariableSpecService.get("RUDDER_DIRECTIVES_SEQUENCE"), bundleVars.directivesUsebundle :: Nil) - ).map(x => (x.spec.name, x)).toMap + buildBundleSequence.prepareBundleVars(nodeId, agentType, nodePolicyMode, globalPolicyMode, container).map { bundleVars => + bundleVars.map(x => (x.spec.name, x)).toMap } } diff --git a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PromiseWriteDataStructures.scala b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PromiseWriteDataStructures.scala index 27f345e0cc..a45afeaa1b 100644 --- a/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PromiseWriteDataStructures.scala +++ b/rudder-core/src/main/scala/com/normation/rudder/services/policies/write/PromiseWriteDataStructures.scala @@ -75,7 +75,8 @@ case class AgentNodeConfiguration( * - the expected reports csv file content */ final case class AgentNodeWritableConfiguration( - paths : NodePromisesPaths + agentType : AgentType + , paths : NodePromisesPaths , preparedTechniques: Seq[PreparedTechnique] , expectedReportsCsv: ExpectedReportsCsv ) diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/Create_file/1.0/Create_file.ps1 b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/Create_file/1.0/Create_file.ps1 new file mode 100644 index 0000000000..1f5b2f3c77 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/Create_file/1.0/Create_file.ps1 @@ -0,0 +1,4 @@ +# +# This file will contain the actual code of the technique for create file, +# as generated by the technique editor. +# \ No newline at end of file diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/dsc-agent/1.0/some-resource.conf b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/dsc-agent/1.0/some-resource.conf new file mode 100644 index 0000000000..911d3962d5 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/dsc-agent/1.0/some-resource.conf @@ -0,0 +1,3 @@ +Blabla bla bla some conf. + +Done. \ No newline at end of file diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/properties.d/properties.json b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/properties.d/properties.json new file mode 100644 index 0000000000..8f2512d7b4 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/properties.d/properties.json @@ -0,0 +1,3 @@ +{ + "properties":{} +} \ No newline at end of file diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-directives.ps1 b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-directives.ps1 new file mode 100644 index 0000000000..ec6d4c5f34 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-directives.ps1 @@ -0,0 +1,27 @@ +##################################################################################### +# Copyright 2017 Normation SAS +##################################################################################### +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, Version 3. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +##################################################################################### + +# This file is the main entry points for the bundle sequence for +# Rudder user directives. It is actually a list of method calls. + +function Call_Directives( + [string] $class = $null + ) { + + Create_file -ReportId 208716db-2675-43b9-ab57-bfbab84346aa@@16d86a56-93ef-49aa-86b7-0d10102e4ea9@@0 -TechniqueName "50-rule-technique-ncf/Create a file" -AuditOnly $false +} diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-system-directives.ps1 b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-system-directives.ps1 new file mode 100644 index 0000000000..d98c5ee4c7 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder-system-directives.ps1 @@ -0,0 +1,39 @@ +##################################################################################### +# Copyright 2017 Normation SAS +##################################################################################### +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, Version 3. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +##################################################################################### + +# This file is the main entry points for the bundle sequence for +# Rudder system directives. It is actually a list of method calls. + +function Call_System_Directives( + [string] $class = $null + ) { + $system_techniques_path = "$rudder_path\system-techniques" + + Write-Verbose ("Loading all system techniques scripts in $system_techniques_path") + + # Get list of all files in ncf folders, and load them + $system_techniques_files = Get-ChildItem -recurse $system_techniques_path -Filter *.ps1 + + $system_techniques_files | ForEach { + . $_.FullName + } + + Write-Verbose ("Loaded all system technique ps1 scripts") + + configure_logger -ReportId dsc-agent@@dsc-agent@@42 +} diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.json b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.json new file mode 100644 index 0000000000..7afb109af9 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.json @@ -0,0 +1 @@ +{ "comment":"for now, an empty file" } diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.ps1 b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.ps1 new file mode 100644 index 0000000000..8be360a5f1 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/node-dsc-with-one-directive/rules/dsc/rudder.ps1 @@ -0,0 +1,8 @@ +[CmdletBinding()] +param ( +[Parameter(Mandatory=$true)] +[string] $action = "run", +[string] $class = $null +) + +Write-Error "Not an actual agent" \ No newline at end of file diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/common/1.0/cf-served.cf b/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/common/1.0/cf-served.cf index f7a8704a89..8e18b926d2 100644 --- a/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/common/1.0/cf-served.cf +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/common/1.0/cf-served.cf @@ -61,12 +61,12 @@ bundle server access_rules "/var/rudder/share/root/" maproot => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, admit => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, - admit_keys => { "MD5=" }; + admit_keys => { "MD5=081cf3aac62624ebbc83be7e23cb104d" }; "/var/rudder/shared-files/root/" maproot => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, admit => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, - admit_keys => { "MD5=" }; + admit_keys => { "MD5=081cf3aac62624ebbc83be7e23cb104d" }; diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/distributePolicy/1.0/nodeslist.json b/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/distributePolicy/1.0/nodeslist.json index a16f556b46..84fffdfc18 100644 --- a/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/distributePolicy/1.0/nodeslist.json +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/root-default-install/distributePolicy/1.0/nodeslist.json @@ -1,7 +1,7 @@ { "root": { "hostname": "server.rudder.local", - "key-hash": "sha256:", + "key-hash": "sha256:cf621af57886d614617b2c8187d1db9d18c93ee0674c9985fb06fa17705ed4f0", "policy-server": "root" } diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/common/1.0/cf-served.cf b/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/common/1.0/cf-served.cf index f7a8704a89..8e18b926d2 100644 --- a/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/common/1.0/cf-served.cf +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/common/1.0/cf-served.cf @@ -61,12 +61,12 @@ bundle server access_rules "/var/rudder/share/root/" maproot => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, admit => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, - admit_keys => { "MD5=" }; + admit_keys => { "MD5=081cf3aac62624ebbc83be7e23cb104d" }; "/var/rudder/shared-files/root/" maproot => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, admit => { host2ip("server.rudder.local"), string_downcase(escape("server.rudder.local")) }, - admit_keys => { "MD5=" }; + admit_keys => { "MD5=081cf3aac62624ebbc83be7e23cb104d" }; diff --git a/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/distributePolicy/1.0/nodeslist.json b/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/distributePolicy/1.0/nodeslist.json index a16f556b46..84fffdfc18 100644 --- a/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/distributePolicy/1.0/nodeslist.json +++ b/rudder-core/src/test/resources/configuration-repository/expected-share/root-with-two-directives/distributePolicy/1.0/nodeslist.json @@ -1,7 +1,7 @@ { "root": { "hostname": "server.rudder.local", - "key-hash": "sha256:", + "key-hash": "sha256:cf621af57886d614617b2c8187d1db9d18c93ee0674c9985fb06fa17705ed4f0", "policy-server": "root" } diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/category.xml b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/category.xml new file mode 100644 index 0000000000..9a37262742 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/category.xml @@ -0,0 +1,25 @@ + + + + DSC Specific system techniques + + This category contains internal Techniques (system) for + DSC agent. + These Techniques should not be used directly. + + true + diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/metadata.xml b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/metadata.xml new file mode 100644 index 0000000000..a2b18afbea --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/metadata.xml @@ -0,0 +1,58 @@ + + + + + System technique for the DSC Agent + true + + + + + rudder-directives.ps1 + + + rudder-system-directives.ps1 + + + + + + rudder.ps1 + false + + + + + + configure_logger + + + + + RUDDER_DIRECTIVES_SEQUENCE + RUDDER_SYSTEM_DIRECTIVES_SEQUENCE + + + + +
+
+
+
+ + + diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-directives.st b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-directives.st new file mode 100644 index 0000000000..7980fbfe30 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-directives.st @@ -0,0 +1,27 @@ +##################################################################################### +# Copyright 2017 Normation SAS +##################################################################################### +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, Version 3. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +##################################################################################### + +# This file is the main entry points for the bundle sequence for +# Rudder user directives. It is actually a list of method calls. + +function Call_Directives( + [string] $class = $null + ) { + +&RUDDER_DIRECTIVES_SEQUENCE& +} diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-system-directives.st b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-system-directives.st new file mode 100644 index 0000000000..b47d238332 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder-system-directives.st @@ -0,0 +1,39 @@ +##################################################################################### +# Copyright 2017 Normation SAS +##################################################################################### +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, Version 3. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +##################################################################################### + +# This file is the main entry points for the bundle sequence for +# Rudder system directives. It is actually a list of method calls. + +function Call_System_Directives( + [string] $class = $null + ) { + $system_techniques_path = "$rudder_path\system-techniques" + + Write-Verbose ("Loading all system techniques scripts in $system_techniques_path") + + # Get list of all files in ncf folders, and load them + $system_techniques_files = Get-ChildItem -recurse $system_techniques_path -Filter *.ps1 + + $system_techniques_files | ForEach { + . $_.FullName + } + + Write-Verbose ("Loaded all system technique ps1 scripts") + +&RUDDER_SYSTEM_DIRECTIVES_SEQUENCE& +} diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder.ps1 b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder.ps1 new file mode 100644 index 0000000000..8be360a5f1 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/rudder.ps1 @@ -0,0 +1,8 @@ +[CmdletBinding()] +param ( +[Parameter(Mandatory=$true)] +[string] $action = "run", +[string] $class = $null +) + +Write-Error "Not an actual agent" \ No newline at end of file diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/some-resource.conf b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/some-resource.conf new file mode 100644 index 0000000000..911d3962d5 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/techniques/dsc-agent/dsc-agent/1.0/some-resource.conf @@ -0,0 +1,3 @@ +Blabla bla bla some conf. + +Done. \ No newline at end of file diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/Create_file.ps1 b/rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/Create_file.ps1 new file mode 100644 index 0000000000..1f5b2f3c77 --- /dev/null +++ b/rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/Create_file.ps1 @@ -0,0 +1,4 @@ +# +# This file will contain the actual code of the technique for create file, +# as generated by the technique editor. +# \ No newline at end of file diff --git a/rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/metadata.xml b/rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/metadata.xml index 3bd36ec304..f954ef5238 100644 --- a/rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/metadata.xml +++ b/rudder-core/src/test/resources/configuration-repository/techniques/ncf_techniques/Create_file/1.0/metadata.xml @@ -2,7 +2,7 @@ Create a file and a directory Create_file - Create_file_rudder_reporting + Create_file_rudder_reporting @@ -12,6 +12,17 @@ true + + + Create_file + + + + + true + + +
diff --git a/rudder-core/src/test/scala/com/normation/cfclerk/domain/TechniqueTest.scala b/rudder-core/src/test/scala/com/normation/cfclerk/domain/TechniqueTest.scala index e6207789a6..f269bbbb51 100644 --- a/rudder-core/src/test/scala/com/normation/cfclerk/domain/TechniqueTest.scala +++ b/rudder-core/src/test/scala/com/normation/cfclerk/domain/TechniqueTest.scala @@ -92,26 +92,30 @@ class TechniqueTest extends Specification { technique.providesExpectedReports == true } - "have bundle list: 'bundle1,bundle2'" in { - technique.bundlesequence.map( _.name ) === Seq("bundle1","bundle2") + "have bundle list: 'bundle1,bundle2' for each agent" in { + technique.agentConfigs.map( _.bundlesequence.map( _.value )) must contain( beEqualTo(Seq("bundle1","bundle2"))).foreach } - "have templates 'tml1, tml2, tml3'" in { - technique.templates.map( _.id.name ) === Seq("tml1", "tml2", "tml3") + "have templates 'tml1, tml2, tml3' for each agent" in { + technique.agentConfigs.map( _.templates.map( _.id.name )) must contain( beEqualTo( Seq("tml1", "tml2", "tml3"))).foreach + } + + def getTemplateByid(technique: Technique, name: String) = { + technique.agentConfigs.head.templates.find( _.id == TechniqueResourceIdByName(technique.id, name)).getOrElse(throw new Exception(s"Test must contain resource '${name}'")) } "'tml1' is included and has default outpath" in { - val tml = technique.templatesMap(TechniqueResourceIdByName(id,"tml1")) + val tml = getTemplateByid(technique, "tml1") tml.included === true and tml.outPath === "foo/1.0/tml1.cf" } "'tml2' is included and has tml2.bar outpath" in { - val tml = technique.templatesMap(TechniqueResourceIdByName(id,"tml2")) + val tml = getTemplateByid(technique, "tml2") tml.included === true and tml.outPath === "tml2.bar" } "'tml3' is not included and has default outpath" in { - val tml = technique.templatesMap(TechniqueResourceIdByName(id,"tml3")) + val tml = getTemplateByid(technique, "tml3") tml.included === false and tml.outPath === "foo/1.0/tml3.cf" } diff --git a/rudder-core/src/test/scala/com/normation/cfclerk/services/DummyTechniqueRepository.scala b/rudder-core/src/test/scala/com/normation/cfclerk/services/DummyTechniqueRepository.scala index 58680ffe29..e9f7cee015 100644 --- a/rudder-core/src/test/scala/com/normation/cfclerk/services/DummyTechniqueRepository.scala +++ b/rudder-core/src/test/scala/com/normation/cfclerk/services/DummyTechniqueRepository.scala @@ -38,26 +38,29 @@ package com.normation.cfclerk.services import com.normation.cfclerk.domain._ -import java.io.{ InputStream, File } +import java.io.InputStream import net.liftweb.common._ import scala.collection.SortedSet +import com.normation.inventory.domain.AgentType class DummyTechniqueRepository(policies: Seq[Technique] = Seq()) extends TechniqueRepository { + def agentCfg(bundle: String) = AgentConfig(AgentType.CfeCommunity, Seq(), Seq(), Seq(BundleName(bundle))) :: Nil + var returnedVariable = collection.mutable.Set[VariableSpec]() - val policy1 = Technique(TechniqueId(TechniqueName("policy1"), TechniqueVersion("1.0")), "policy1", "", Seq(), Seq(), Seq(Bundle("one")), TrackerVariableSpec(), SectionSpec(name="root", children=Seq(InputVariableSpec("$variable1", "a variable1"))), None) + val policy1 = Technique(TechniqueId(TechniqueName("policy1"), TechniqueVersion("1.0")), "policy1", "", agentCfg("one"), TrackerVariableSpec(), SectionSpec(name="root", children=Seq(InputVariableSpec("$variable1", "a variable1"))), None) val sections = SectionSpec(name="root", children=Seq(InputVariableSpec("$variable2", "a variable2", multivalued = true), InputVariableSpec("$variable22", "a variable22"))) - val policy2 = Technique(TechniqueId(TechniqueName("policy2"), TechniqueVersion("1.0")), "policy2", "", Seq(), Seq(), Seq(Bundle("two")), TrackerVariableSpec(), sections, None) + val policy2 = Technique(TechniqueId(TechniqueName("policy2"), TechniqueVersion("1.0")), "policy2", "", agentCfg("two"), TrackerVariableSpec(), sections, None) val sections3 = SectionSpec(name="root", children=Seq(InputVariableSpec("$variable3", "a variable3"))) - val policy3 = Technique(TechniqueId(TechniqueName("policy3"), TechniqueVersion("1.0")), "policy3", "", Seq(), Seq(), Seq(Bundle("three")), TrackerVariableSpec(), sections3, None) + val policy3 = Technique(TechniqueId(TechniqueName("policy3"), TechniqueVersion("1.0")), "policy3", "", agentCfg("three"), TrackerVariableSpec(), sections3, None) val sections4 = SectionSpec(name="root", children=Seq(InputVariableSpec("$variable4", "an variable4"))) - val policy4 = Technique(TechniqueId(TechniqueName("policy4"), TechniqueVersion("1.0")), "policy4", "", Seq(), Seq(), Seq(Bundle("four")), TrackerVariableSpec(), sections4, None) + val policy4 = Technique(TechniqueId(TechniqueName("policy4"), TechniqueVersion("1.0")), "policy4", "", agentCfg("four"), TrackerVariableSpec(), sections4, None) val sectionsFoo = SectionSpec(name="root", children=Seq(InputVariableSpec("$bar", "bar"))) - val foo = Technique(TechniqueId(TechniqueName("foo"), TechniqueVersion("1.0")), "foo", "", Seq(), Seq(), Seq(Bundle("foo")), TrackerVariableSpec(), sectionsFoo, None) + val foo = Technique(TechniqueId(TechniqueName("foo"), TechniqueVersion("1.0")), "foo", "", agentCfg("foo"), TrackerVariableSpec(), sectionsFoo, None) val policyMap = Map(policy1.id -> policy1, policy2.id -> policy2, diff --git a/rudder-core/src/test/scala/com/normation/cfclerk/services/JGitPackageReaderTest.scala b/rudder-core/src/test/scala/com/normation/cfclerk/services/JGitPackageReaderTest.scala index af144935bf..c606a86497 100644 --- a/rudder-core/src/test/scala/com/normation/cfclerk/services/JGitPackageReaderTest.scala +++ b/rudder-core/src/test/scala/com/normation/cfclerk/services/JGitPackageReaderTest.scala @@ -188,7 +188,7 @@ trait JGitPackageReaderSpec extends Specification with Loggable with AfterAll { "...and version 2.0" in techniques(1).version === TechniqueVersion("2.0") ".... that DOES NOT provide expected_reports.csv" in infos.techniques(techniques(1).name)(techniques(1).version).providesExpectedReports === false "...with 3 templates" in { - infos.techniques(techniques(0).name)(techniques(0).version).templates.toSet === Set( + infos.techniques(techniques(0).name)(techniques(0).version).agentConfigs(0).templates.toSet === Set( TechniqueTemplate(tmlId, s"p1_1/1.0/${tmlId.name}.cf", true) , TechniqueTemplate(templateId, s"bob.txt", false) , TechniqueTemplate(template2Id, s"p1_1/1.0/${template2Id.name}.cf", true) @@ -204,7 +204,7 @@ trait JGitPackageReaderSpec extends Specification with Loggable with AfterAll { assertResourceContent(template2Id, true, template2Content) } "...with 2 files" in { - infos.techniques(techniques(0).name)(techniques(0).version).files.toSet === Set( + infos.techniques(techniques(0).name)(techniques(0).version).agentConfigs(0).files.toSet === Set( TechniqueFile(file1, s"p1_1/1.0/${file1.name}", false) , TechniqueFile(file2, s"file2", false) ) diff --git a/rudder-core/src/test/scala/com/normation/rudder/services/policies/NodeConfigData.scala b/rudder-core/src/test/scala/com/normation/rudder/services/policies/NodeConfigData.scala index 8970ee8e81..a4f10e9771 100644 --- a/rudder-core/src/test/scala/com/normation/rudder/services/policies/NodeConfigData.scala +++ b/rudder-core/src/test/scala/com/normation/rudder/services/policies/NodeConfigData.scala @@ -94,6 +94,10 @@ import com.normation.rudder.services.policies.write.Cf3PolicyDraft import org.joda.time.DateTime import scala.collection.SortedMap import scala.language.implicitConversions +import com.normation.inventory.domain.Windows +import com.normation.inventory.domain.Windows2012 +import com.normation.inventory.domain.AgentType +import com.normation.inventory.domain.Certificate /* * This file is a container for testing data that are a little boring to @@ -102,6 +106,18 @@ import scala.language.implicitConversions */ object NodeConfigData { + //a valid, not used pub key + //cfengine key hash is: 081cf3aac62624ebbc83be7e23cb104d + val PUBKEY = +"""-----BEGIN RSA PUBLIC KEY----- +MIIBCAKCAQEAlntroa72gD50MehPoyp6mRS5fzZpsZEHu42vq9KKxbqSsjfUmxnT +Rsi8CDvBt7DApIc7W1g0eJ6AsOfV7CEh3ooiyL/fC9SGATyDg5TjYPJZn3MPUktg +YBzTd1MMyZL6zcLmIpQBH6XHkH7Do/RxFRtaSyicLxiO3H3wapH20TnkUvEpV5Qh +zUkNM8vHZuu3m1FgLrK5NCN7BtoGWgeyVJvBMbWww5hS15IkCRuBkAOK/+h8xe2f +hMQjrt9gW2qJpxZyFoPuMsWFIaX4wrN7Y8ZiN37U2q1G11tv2oQlJTQeiYaUnTX4 +z5VEb9yx2KikbWyChM1Akp82AV5BzqE80QIBIw== +-----END RSA PUBLIC KEY-----""" + val emptyNodeReportingConfiguration = ReportingConfiguration(None,None) val id1 = NodeId("node1") @@ -133,7 +149,7 @@ object NodeConfigData { , List("127.0.0.1", "192.168.0.100") , DateTime.now , UndefinedKey - , Seq(AgentInfo(CfeCommunity, Some(AgentVersion("4.0.0")), PublicKey("test"))) + , Seq(AgentInfo(CfeCommunity, Some(AgentVersion("4.0.0")), PublicKey(PUBKEY))) , rootId , rootAdmin , Set( //by default server roles for root @@ -172,7 +188,7 @@ object NodeConfigData { , List("192.168.0.10") , DateTime.now , UndefinedKey - , Seq(AgentInfo(CfeCommunity, Some(AgentVersion("4.0.0")), PublicKey("test"))) + , Seq(AgentInfo(CfeCommunity, Some(AgentVersion("4.0.0")), PublicKey(PUBKEY))) , rootId , admin1 , Set() @@ -217,6 +233,69 @@ object NodeConfigData { val node2Node = node1Node.copy(id = id2, name = id2.value) val node2 = node1.copy(node = node2Node, hostname = hostname2, policyServerId = node1.id ) + val dscNode1Node = Node ( + NodeId("node-dsc") + , "node-dsc" + , "" + , false + , false + , true //is policy server + , DateTime.now + , emptyNodeReportingConfiguration + , Seq() + , None + ) + + val dscNode1 = NodeInfo ( + dscNode1Node + , "node-dsc.localhost" + , Some(MachineInfo(MachineUuid("machine1"), VirtualMachineType(VirtualBox), None, None)) + , Windows(Windows2012, "Windows 2012 youpla boom", new Version("2012"), Some("sp1"), new Version("win-kernel-2012")) + , List("192.168.0.5") + , DateTime.now + , UndefinedKey + , Seq(AgentInfo(AgentType.Dsc, Some(AgentVersion("5.0.0")), Certificate("windows-node-dsc-certificate"))) + , rootId + , admin1 + , Set() + , None + , Some(MemorySize(1460132)) + , None + ) + + val dscInventory1: NodeInventory = NodeInventory( + NodeSummary( + dscNode1.id + , AcceptedInventory + , dscNode1.localAdministratorAccountName + , dscNode1.hostname + , dscNode1.osDetails + , dscNode1.policyServerId + , UndefinedKey + ) + , name = None + , description = None + , ram = None + , swap = None + , inventoryDate = None + , receiveDate = None + , archDescription = None + , lastLoggedUser = None + , lastLoggedUserTime = None + , agents = Seq() + , serverIps = Seq() + , machineId = None //if we want several ids, we would have to ass an "alternate machine" field + , softwareIds = Seq() + , accounts = Seq() + , environmentVariables = Seq(EnvironmentVariable("THE_VAR", Some("THE_VAR value!"))) + , processes = Seq() + , vms = Seq() + , networks = Seq() + , fileSystems = Seq() + , serverRoles = Set() + ) + + val allNodesInfo = Map( rootId -> root, node1.id -> node1, node2.id -> node2) val defaultModesConfig = NodeModeConfig( @@ -327,7 +406,7 @@ object NodeConfigData { implicit def toDID(id: String) = DirectiveId(id) implicit def toRID(id: String) = RuleId(id) implicit def toRCID(id: String) = RuleCategoryId(id) - val t1 = Technique(("t1", "1.0"), "t1", "t1", Seq(), Seq(), Seq(), TrackerVariableSpec(), SectionSpec("root"), None) + val t1 = Technique(("t1", "1.0"), "t1", "t1", Nil, TrackerVariableSpec(), SectionSpec("root"), None) val d1 = Directive("d1", "1.0", Map("foo1" -> Seq("bar1")), "d1", "d1", None) val d2 = Directive("d2", "1.0", Map("foo2" -> Seq("bar2")), "d2", "d2", Some(PolicyMode.Enforce)) val d3 = Directive("d3", "1.0", Map("foo3" -> Seq("bar3")), "d3", "d3", Some(PolicyMode.Audit)) diff --git a/rudder-core/src/test/scala/com/normation/rudder/services/policies/RuleValServiceTest.scala b/rudder-core/src/test/scala/com/normation/rudder/services/policies/RuleValServiceTest.scala index c3db5e712f..10fd2ea32e 100644 --- a/rudder-core/src/test/scala/com/normation/rudder/services/policies/RuleValServiceTest.scala +++ b/rudder-core/src/test/scala/com/normation/rudder/services/policies/RuleValServiceTest.scala @@ -116,9 +116,7 @@ class RuleValServiceTest extends Specification { id , "meta" + id , "" - , Seq() - , Seq() - , Seq() + , Nil , TrackerVariableSpec(None) , makeRootSectionSpec , None diff --git a/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PolicyInstanceAgregationTest.scala b/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PolicyInstanceAgregationTest.scala index 115687bae6..be7e873301 100644 --- a/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PolicyInstanceAgregationTest.scala +++ b/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PolicyInstanceAgregationTest.scala @@ -49,8 +49,6 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.BlockJUnit4ClassRunner import com.normation.cfclerk.services.DummyTechniqueRepository -import scala.collection.immutable.Set -import scala.language.implicitConversions import org.joda.time.DateTime @@ -83,9 +81,7 @@ class DirectiveAgregationTest { activeTechniqueId1 , "name" , "DESCRIPTION" - , Seq() - , Seq() - , Seq() + , Nil , trackerVariableSpec , SectionSpec(name="root", children=Seq()) , None @@ -95,9 +91,7 @@ class DirectiveAgregationTest { activeTechniqueId2 , "name" , "DESCRIPTION" - , Seq() - , Seq() - , Seq() + , Nil , trackerVariableSpec , SectionSpec(name="root", children=Seq()) , None @@ -105,9 +99,11 @@ class DirectiveAgregationTest { ) ) ) + val systemVariableServiceSpec = new SystemVariableSpecServiceImpl() val prepareTemplate = new PrepareTemplateVariablesImpl( techniqueRepository - , new SystemVariableSpecServiceImpl() + , systemVariableServiceSpec + , new BuildBundleSequence(systemVariableServiceSpec) ) def createDirectiveWithBinding(activeTechniqueId:TechniqueId, i: Int): Cf3PolicyDraft = { @@ -172,7 +168,7 @@ class DirectiveAgregationTest { def arrayedDirectiveTest() { val newTechniqueId = TechniqueId(TechniqueName("name"), TechniqueVersion("1.0")) - def newTechnique = Technique(newTechniqueId, "tech" + newTechniqueId, "", Seq(), Seq(), Seq(), TrackerVariableSpec(), SectionSpec("plop"), None, Set(), None) + def newTechnique = Technique(newTechniqueId, "tech" + newTechniqueId, "", Nil, TrackerVariableSpec(), SectionSpec("plop"), None, Set(), None) val instance = new Cf3PolicyDraft("id", newTechnique, DateTime.now, Map(), trackerVariable, priority = 0, serial = 0, ruleOrder = BundleOrder("r"), directiveOrder = BundleOrder("d"), overrides = Set(), policyMode = None, isSystem = false) diff --git a/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariableTest.scala b/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariableTest.scala index 37b4e1d047..9412401f03 100644 --- a/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariableTest.scala +++ b/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/PrepareTemplateVariableTest.scala @@ -40,37 +40,37 @@ import org.junit.runner._ import org.specs2.mutable._ import org.specs2.runner._ import com.normation.rudder.services.policies.write.BuildBundleSequence._ -import com.normation.cfclerk.domain.Bundle +import com.normation.cfclerk.domain.BundleName import com.normation.rudder.domain.policies.PolicyMode @RunWith(classOf[JUnitRunner]) class PrepareTemplateVariableTest extends Specification { val bundles = Seq( - ("Global configuration for all nodes/20. Install jdk version 1.0" , Bundle("Install_jdk_rudder_reporting")) - , ("Global configuration for all nodes/RUG / YaST package manager configuration (ZMD)", Bundle("check_zmd_settings")) - , ("""Nodes only/Name resolution version "3.0" and counting""" , Bundle("check_dns_configuration")) - , (raw"""Nodes only/Package \"management\" for Debian""" , Bundle("check_apt_package_installation")) - , (raw"""Nodes only/Package \\"management\\" for Debian - again""" , Bundle("check_apt_package_installation2")) - ).map { case(x,y) => TechniqueBundles(Promiser(x), Nil, y::Nil, Nil, false, false, PolicyMode.Enforce) } + ("Global configuration for all nodes/20. Install jdk version 1.0" , Bundle(ReportId("not used"), BundleName("Install_jdk_rudder_reporting"))) + , ("Global configuration for all nodes/RUG / YaST package manager configuration (ZMD)", Bundle(ReportId("not used"), BundleName("check_zmd_settings"))) + , ("""Nodes only/Name resolution version "3.0" and counting""" , Bundle(ReportId("not used"), BundleName("check_dns_configuration"))) + , (raw"""Nodes only/Package \"management\" for Debian""" , Bundle(ReportId("not used"), BundleName("check_apt_package_installation"))) + , (raw"""Nodes only/Package \\"management\\" for Debian - again""" , Bundle(ReportId("not used"), BundleName("check_apt_package_installation2"))) + ).map { case(x,y) => TechniqueBundles(Directive(x), Nil, y::Nil, Nil, false, false, PolicyMode.Enforce) } // Ok, now I can test "Preparing the string for writting usebundle of directives" should { "correctly write nothing at all when the list of bundle is emtpy" in { - formatMethodsUsebundle(Seq()) === "" + CfengineBundleVariables.formatMethodsUsebundle(Seq()) === List("") } "write exactly - including escaped quotes" in { //spaces inserted at the begining of promises in rudder_directives.cf are due to string template, not the formated string - strange - formatMethodsUsebundle(bundles) === -raw""""Global configuration for all nodes/20. Install jdk version 1.0" usebundle => Install_jdk_rudder_reporting; + CfengineBundleVariables.formatMethodsUsebundle(bundles) === +List(raw""""Global configuration for all nodes/20. Install jdk version 1.0" usebundle => Install_jdk_rudder_reporting; |"Global configuration for all nodes/RUG / YaST package manager configuration (ZMD)" usebundle => check_zmd_settings; |"Nodes only/Name resolution version \"3.0\" and counting" usebundle => check_dns_configuration; |"Nodes only/Package \\\"management\\\" for Debian" usebundle => check_apt_package_installation; - |"Nodes only/Package \\\\\"management\\\\\" for Debian - again" usebundle => check_apt_package_installation2;""".stripMargin + |"Nodes only/Package \\\\\"management\\\\\" for Debian - again" usebundle => check_apt_package_installation2;""".stripMargin) } } diff --git a/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/WriteSystemTechniquesTest.scala b/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/WriteSystemTechniquesTest.scala index f8c874b2d0..91dcb34595 100644 --- a/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/WriteSystemTechniquesTest.scala +++ b/rudder-core/src/test/scala/com/normation/rudder/services/policies/write/WriteSystemTechniquesTest.scala @@ -100,6 +100,7 @@ import com.normation.rudder.domain.policies.PolicyModeOverrides import com.normation.BoxSpecMatcher import com.normation.rudder.services.policies.NodeConfigData import com.normation.rudder.domain.nodes.NodeInfo +import com.normation.cfclerk.domain.Variable /** * Details of tests executed in each instances of @@ -110,6 +111,14 @@ import com.normation.rudder.domain.nodes.NodeInfo @RunWith(classOf[JUnitRunner]) class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpecMatcher with ContentMatchers with AfterAll { + //make tests more similar than default rudder install + val hookIgnore = """.swp, ~, .bak, + .cfnew , .cfsaved , .cfedited, .cfdisabled, .cfmoved, + .dpkg-old, .dpkg-dist, .dpkg-new, .dpkg-tmp, + .disable , .disabled , _disable , _disabled, + .ucf-old , .ucf-dist , .ucf-new , + .rpmnew , .rpmsave , .rpmorig""".split(",").map( _.trim).toList + //just a little sugar to stop hurting my eyes with new File(blablab, plop) implicit class PathString(root: String) { def /(child: String) = new File(root, child) @@ -131,10 +140,11 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec val EXPECTED_SHARE = configurationRepositoryRoot/"expected-share" val variableSpecParser = new VariableSpecParser + val systemVariableServiceSpec = new SystemVariableSpecServiceImpl() val policyParser: TechniqueParser = new TechniqueParser( - variableSpecParser, - new SectionSpecParser(variableSpecParser), - new SystemVariableSpecServiceImpl + variableSpecParser + , new SectionSpecParser(variableSpecParser) + , systemVariableServiceSpec ) val reader = new GitTechniqueReader( policyParser @@ -154,7 +164,6 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec override def setAuthorizedNetworks(policyServerId:NodeId, networks:Seq[String], modId: ModificationId, actor:EventActor) = ??? override def getAuthorizedNetworks(policyServerId:NodeId) : Box[Seq[String]] = Full(List("192.168.49.0/24")) } - val systemVariableServiceSpec = new SystemVariableSpecServiceImpl() val systemVariableService = new SystemVariableServiceImpl( systemVariableServiceSpec , policyServerManagement @@ -192,6 +201,7 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec val prepareTemplateVariable = new PrepareTemplateVariablesImpl( techniqueRepository , systemVariableServiceSpec + , new BuildBundleSequence(systemVariableServiceSpec) ) @@ -220,8 +230,8 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec , logNodeConfig , prepareTemplateVariable , new FillTemplatesService() - , "/opt/rudder/etc/hooks.d" - , Nil + , "/we-don-t-want-hooks-here" + , hookIgnore ) (rootGeneratedPromisesDir, promiseWritter) @@ -271,14 +281,16 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec // set up root node configuration ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - import com.normation.rudder.services.policies.NodeConfigData.{root, node1, rootNodeConfig} + import com.normation.rudder.services.policies.NodeConfigData.{root, node1, rootNodeConfig, dscNode1} //a test node - CFEngine val nodeId = NodeId("c8813416-316f-4307-9b6a-ca9c109a9fb0") val cfeNode = node1.copy(node = node1.node.copy(id = nodeId, name = nodeId.value)) + val dscNode = dscNode1.copy(node = dscNode1.node.copy(id = nodeId, name = nodeId.value)) val allNodesInfo_rootOnly = Map(root.id -> root) val allNodesInfo_cfeNode = Map(root.id -> root, cfeNode.id -> cfeNode) + val allNodesInfo_dscNode = Map(root.id -> root, dscNode.id -> dscNode) //the group lib val emptyGroupLib = FullNodeGroupCategory( @@ -560,6 +572,28 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec ////////////// + val dscAgentTechnique = techniqueRepository.get(TechniqueId(TechniqueName("dsc-agent"), TechniqueVersion("1.0"))).getOrElse(throw new RuntimeException("Bad init for test")) + val dscAgentVariables = { + val spec = dscAgentTechnique.getAllVariableSpecs.map(s => (s.name, s)).toMap + Seq[Variable]( +// spec("FOO").toVariable(Seq("Some value for the variable.")) + ).map(v => (v.spec.name, v)).toMap + } + val dscAgent = Cf3PolicyDraft( + id = Cf3PolicyDraftId(RuleId("dsc-agent"), DirectiveId("dsc-agent")) + , technique = dscAgentTechnique + , techniqueUpdateTime = DateTime.now + , variableMap = dscAgentVariables + , trackerVariable = dscAgentTechnique.trackerVariableSpec.toVariable(Seq()) + , priority = 5 + , serial = 42 + , modificationDate= DateTime.now + , ruleOrder = BundleOrder("dsc-agent") + , directiveOrder = BundleOrder("dsc-agent") + , overrides = Set() + , policyMode = None + , isSystem = true + ) // Allows override in policy mode, but default to audit val globalPolicyMode = GlobalPolicyMode(PolicyMode.Audit, PolicyModeOverrides.Always) @@ -588,6 +622,11 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec , parameters = Set(ParameterForConfiguration(ParameterName("rudder_file_edit_header"), "### Managed by Rudder, edit with care ###")) ) + val dscNodeConfig = NodeConfigData.node1NodeConfig.copy( + nodeInfo = cfeNode + , parameters = Set(ParameterForConfiguration(ParameterName("rudder_file_edit_header"), "### Managed by Rudder, edit with care ###")) + ) + // A global list of files to ignore because variable content (like timestamp) def filterGeneratedFile(f: File): Boolean = { f.getName match { @@ -622,7 +661,7 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec } "A root node, with no node connected" should { - def writeNodeConfigWithUserDirectives(promiseWritter: Cf3PromisesFileWriterService, userDrafts: Cf3PolicyDraft*) = { + def writeNodeConfigWithUserDirectives(promiseWritter: PolicyWriterService, userDrafts: Cf3PolicyDraft*) = { val rnc = baseRootNodeConfig.copy( policyDrafts = baseRootNodeConfig.policyDrafts ++ userDrafts ) @@ -648,42 +687,6 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec } } - "A CFEngine node, with two directives" should { - - val rnc = rootNodeConfig.copy( - policyDrafts = Set(common(root.id, allNodesInfo_cfeNode), serverRole, distributePolicy, inventoryAll) - , nodeContext = getSystemVars(root, allNodesInfo_cfeNode, groupLib) - , parameters = Set(ParameterForConfiguration(ParameterName("rudder_file_edit_header"), "### Managed by Rudder, edit with care ###")) - ) - - val cfeNC = cfeNodeConfig.copy( - nodeInfo = cfeNode - , policyDrafts = Set(common(cfeNode.id, allNodesInfo_cfeNode), inventoryAll, pkg, ncf1) - , nodeContext = getSystemVars(cfeNode, allNodesInfo_cfeNode, groupLib) - ) - - "correctly get the expected policy files" in { - val (rootPath, writter) = getPromiseWritter("cfe-node") - // Actually write the promise files for the root node - val writen = writter.writeTemplate( - root.id - , Set(root.id, cfeNode.id) - , Map(root.id -> rnc, cfeNode.id -> cfeNC) - , Map(root.id -> NodeConfigId("root-cfg-id"), cfeNode.id -> NodeConfigId("cfe-node-cfg-id")) - , Map(), globalPolicyMode, DateTime.now - ) - - (writen mustFull) and - compareWith(rootPath.getParentFile/cfeNode.id.value, "node-cfe-with-two-directives", - """.*rudder_common_report\("ntpConfiguration".*@@.*""" //clock reports - :: """.*add:default:==:.*""" //rpm reports - :: Nil - ) - } - - - } - "rudder-group.st template" should { def getRootNodeConfig(groupLib: FullNodeGroupCategory) = { @@ -734,4 +737,74 @@ class WriteSystemTechniquesTest extends Specification with Loggable with BoxSpec rootPath/"common/1.0/rudder-groups.cf" must haveSameLinesAs(EXPECTED_SHARE/"test-rudder-groups/some-groups.cf") } } + + "A CFEngine node, with two directives" should { + + val rnc = rootNodeConfig.copy( + policyDrafts = Set(common(root.id, allNodesInfo_cfeNode), serverRole, distributePolicy, inventoryAll) + , nodeContext = getSystemVars(root, allNodesInfo_cfeNode, groupLib) + , parameters = Set(ParameterForConfiguration(ParameterName("rudder_file_edit_header"), "### Managed by Rudder, edit with care ###")) + ) + + val cfeNC = cfeNodeConfig.copy( + nodeInfo = cfeNode + , policyDrafts = Set(common(cfeNode.id, allNodesInfo_cfeNode), inventoryAll, pkg, ncf1) + , nodeContext = getSystemVars(cfeNode, allNodesInfo_cfeNode, groupLib) + ) + + "correctly get the expected policy files" in { + val (rootPath, writter) = getPromiseWritter("cfe-node") + // Actually write the promise files for the root node + val writen = writter.writeTemplate( + root.id + , Set(root.id, cfeNode.id) + , Map(root.id -> rnc, cfeNode.id -> cfeNC) + , Map(root.id -> NodeConfigId("root-cfg-id"), cfeNode.id -> NodeConfigId("cfe-node-cfg-id")) + , Map(), globalPolicyMode, DateTime.now + ) + + (writen mustFull) and + compareWith(rootPath.getParentFile/cfeNode.id.value, "node-cfe-with-two-directives", + """.*rudder_common_report\("ntpConfiguration".*@@.*""" //clock reports + :: """.*add:default:==:.*""" //rpm reports + :: Nil + ) + } + } + + "A DSC node, with one directives" should { + + val rnc = rootNodeConfig.copy( + policyDrafts = Set(common(root.id, allNodesInfo_cfeNode), serverRole, distributePolicy, inventoryAll) + , nodeContext = getSystemVars(root, allNodesInfo_cfeNode, groupLib) + , parameters = Set(ParameterForConfiguration(ParameterName("rudder_file_edit_header"), "### Managed by Rudder, edit with care ###")) + ) + + val dscNC = cfeNodeConfig.copy( + nodeInfo = dscNode + , policyDrafts = Set(dscAgent, ncf1) + , nodeContext = getSystemVars(dscNode, allNodesInfo_dscNode, groupLib) + ) + + "correctly get the expected policy files" in { + val (rootPath, writter) = getPromiseWritter("dsc-node") + // Actually write the promise files for the root node + val writen = writter.writeTemplate( + root.id + , Set(root.id, dscNode.id) + , Map(root.id -> rnc, dscNode.id -> dscNC) + , Map(root.id -> NodeConfigId("root-cfg-id"), dscNode.id -> NodeConfigId("dsc-node-cfg-id")) + , Map(), globalPolicyMode, DateTime.now + ) + + (writen mustFull) and + compareWith(rootPath.getParentFile/dscNode.id.value, "node-dsc-with-one-directive", + """.*rudder_common_report\("ntpConfiguration".*@@.*""" //clock reports + :: """.*add:default:==:.*""" //rpm reports + :: Nil + ) + } + + + } } diff --git a/rudder-web/src/main/scala/bootstrap/liftweb/AppConfig.scala b/rudder-web/src/main/scala/bootstrap/liftweb/AppConfig.scala index 6f014e0352..0cd67fb9aa 100644 --- a/rudder-web/src/main/scala/bootstrap/liftweb/AppConfig.scala +++ b/rudder-web/src/main/scala/bootstrap/liftweb/AppConfig.scala @@ -130,6 +130,7 @@ import com.normation.rudder.web.rest.compliance.ComplianceAPIService import com.normation.rudder.services.policies.write.Cf3PromisesFileWriterServiceImpl import com.normation.rudder.services.policies.write.PathComputerImpl import com.normation.rudder.services.policies.write.PrepareTemplateVariablesImpl +import com.normation.rudder.services.policies.write.BuildBundleSequence import com.typesafe.config.ConfigException import org.apache.commons.io.FileUtils import com.normation.templates.FillTemplatesService @@ -139,6 +140,7 @@ import com.normation.rudder.services.quicksearch.FullQuickSearchService import com.normation.rudder.db.Doobie import com.normation.rudder.web.rest.settings.SettingsAPI8 import com.normation.rudder.web.rest.sharedFiles.SharedFilesAPI +import com.normation.rudder.web.rest.node.NodeApiService2 /** * Define a resource for configuration. @@ -1416,7 +1418,7 @@ object RudderConfig extends Loggable { techniqueRepositoryImpl , pathComputer , new NodeConfigurationLoggerImpl(RUDDER_DEBUG_NODE_CONFIGURATION_PATH) - , new PrepareTemplateVariablesImpl(techniqueRepositoryImpl, systemVariableSpecService) + , new PrepareTemplateVariablesImpl(techniqueRepositoryImpl, systemVariableSpecService, new BuildBundleSequence(systemVariableSpecService)) , new FillTemplatesService() , HOOKS_D , HOOKS_IGNORE_SUFFIXES @@ -1524,8 +1526,7 @@ object RudderConfig extends Loggable { } private[this] lazy val nodeConfigurationServiceImpl: NodeConfigurationService = new NodeConfigurationServiceImpl( - rudderCf3PromisesFileWriterService - , new LdapNodeConfigurationHashRepository(rudderDit, rwLdap) + new LdapNodeConfigurationHashRepository(rudderDit, rwLdap) ) // private[this] lazy val licenseService: CfeEnterpriseLicenseService = new CfeEnterpriseLicenseServiceImpl(licenseRepository, ldapNodeConfigurationRepository, RUDDER_DIR_LICENSESFOLDER) private[this] lazy val reportingServiceImpl = new CachedReportingServiceImpl(