Fixes #19650: Need a migration script about changes in system directi…

…ves, groups and rules

webapp/sources/rudder/rudder-core/src/main/resources/ldap/init-policy-server.ldif

@@ -220,9 +220,8 @@ isSystem: TRUE
 directivePriority: 0
 directiveVariable: OWNER[0]:${rudder.node.admin}
 directiveVariable: UUID[0]:${rudder.node.id}
-directiveVariable: POLICYSERVER[0]:%%POLICY_SERVER_HOSTNAME%%
-directiveVariable: POLICYSERVER_ID[0]:root
-directiveVariable: POLICYSERVER_ADMIN[0]:root
+directiveVariable: POLICYSERVER_ID[0]:${rudder.node.policyserver.id}
+directiveVariable: POLICYSERVER_ADMIN[0]:${rudder.node.policyserver.admin}
 
 #######################################################################################################################
 ## Allowed networks for root server: init corresponding setting in ou=Application Properties

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/logger/MigrationLogger.scala

@@ -37,8 +37,9 @@
 
 package com.normation.rudder.domain.logger
 
-import org.slf4j.LoggerFactory
+import com.normation.NamedZioLogger
 
+import org.slf4j.LoggerFactory
 import com.normation.rudder.domain.Constants.XML_CURRENT_FILE_FORMAT
 import com.normation.rudder.migration.MigrableEntity
 
@@ -66,3 +67,7 @@ final case class MigrationLogger(
     _logger.debug(s"Successfully migrated ${seq.size} eventlogs to format ${goal}")
   }
 }
+
+object MigrationLoggerPure extends NamedZioLogger {
+  def loggerName = "migration"
+}

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/domain/policies/RuleTarget.scala

@@ -326,7 +326,10 @@ object RuleTarget extends Loggable {
         unserJson(parse(s))
       } catch {
         case e : Exception =>
-          logger.error(s"Error when trying to read the following serialized Rule target as a composite target: '${s}'. Reported parsing error cause was: ${e.getMessage}")
+          // these two target were removed in 7.0, we don't want error log on them.
+          if(s != "special:all_servers_with_role" && s != "special:all_servers_with_role") {
+            logger.error(s"Error when trying to read the following serialized Rule target as a composite target: '${s}'. Reported parsing error cause was: ${e.getMessage}")
+          }
           None
       }
     )

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPDirectiveRepository.scala

@@ -622,7 +622,7 @@ class WoLDAPDirectiveRepository(
   private[this] def internalSaveDirective(inActiveTechniqueId:ActiveTechniqueId,directive:Directive, modId: ModificationId, actor:EventActor, reason:Option[String], systemCall:Boolean) : IOResult[Option[DirectiveSaveDiff]] = {
     userLibMutex.writeLock(for {
       con         <- ldap
-      uptEntry    <- getUPTEntry(con, inActiveTechniqueId, "1.1").notOptional(s"Can not find the User Policy Entry with id '${inActiveTechniqueId.value}' to add directive '${directive.id.uid.value}'")
+      uptEntry    <- getUPTEntry(con, inActiveTechniqueId, "1.1").notOptional(s"Can not find the active technique with id '${inActiveTechniqueId.value}' to add directive '${directive.id.uid.value}'")
       canAdd      <- getDirectiveEntry(con, directive.id).flatMap { //check if the directive already exists elsewhere
                        case None          => None.succeed
                        case Some(otherPi) =>
@@ -651,7 +651,7 @@ class WoLDAPDirectiveRepository(
       piEntry           =  mapper.userDirective2Entry(directive, uptEntry.dn)
       result            <- con.save(piEntry, true)
       //for log event - perhaps put that elsewhere ?
-      activeTechnique   <- getActiveTechniqueByActiveTechnique(inActiveTechniqueId).notOptional(s"Can not find the User Policy Entry with id '${inActiveTechniqueId.value}' to add directive '${directive.id.uid.value}'")
+      activeTechnique   <- getActiveTechniqueByActiveTechnique(inActiveTechniqueId).notOptional(s"Can not find active technique Entry with id '${inActiveTechniqueId.value}' to add directive '${directive.id.uid.value}'")
       activeTechniqueId =  TechniqueId(activeTechnique.techniqueName,directive.techniqueVersion)
       technique         <- techniqueRepository.get(activeTechniqueId).notOptional(s"Can not find the technique with ID '${activeTechniqueId.debugString}'")
       optDiff           <- (diffMapper.modChangeRecords2DirectiveSaveDiff(
@@ -948,7 +948,7 @@ class WoLDAPDirectiveRepository(
                               }
                             }
       categoryEntry      <- getCategoryEntry(con, categoryId, "1.1").notOptional(s"Category entry with ID '${categoryId.value}' was not found")
-      newActiveTechnique =  ActiveTechnique(ActiveTechniqueId(uuidGen.newUuid),techniqueName, versions.map(x => x -> DateTime.now()).toMap)
+      newActiveTechnique =  ActiveTechnique(ActiveTechniqueId(techniqueName.value),techniqueName, versions.map(x => x -> DateTime.now()).toMap)
       uptEntry           =  mapper.activeTechnique2Entry(newActiveTechnique,categoryEntry.dn)
       result             <- con.save(uptEntry, true)
       // a new active technique is never system, see constructor call, using defvault value,

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPGitRevisionProvider.scala

@@ -93,7 +93,8 @@ class LDAPGitRevisionProvider(
         logPure.error(s"Error when trying to read persisted version of the current technique " +
           s"reference library revision to use. Using the last available from Git. Error was: ${err.fullMsg}") *> setID
     , res => res match {
-      case Some(id) => IOResult.effect(ObjectId.fromString(id))
+      case Some(id) =>
+        IOResult.effect(ObjectId.fromString(id))
       case None =>
         logPure.info("No persisted version of the current technique reference library revision " +
           "to use where found, init to last available from Git repository") *> setID

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/nodes/NodeInfoService.scala

@@ -126,7 +126,7 @@ trait NodeInfoService {
 
 
   /**
-   * Return the number of managed (ie non policy server, no rudder role )nodes.
+   * Return the number of managed (ie non policy server, no rudder role nodes.
    * Implementation of that method must as efficient as possible.
    * It can't fails (implementation must use a sane default if backend is not accessible,
    * or cache the information)

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/policies/TechniqueAcceptationDatetimeUpdater.scala

@@ -278,26 +278,24 @@ class TechniqueAcceptationUpdater(
                                         //ignore ? => do nothing
                                         UIO.unit
                                       case Some(t) =>
-                                        //if the technique is system, we must not add it to UserTechniqueLib, the process
-                                        //must be handled by a dedicated action. Only update system techniques!
+                                        val referenceId = t.head._2.id
+
+                                        //if the technique is system, we must be careful: the category is not "system" like in fs
+                                        // but "Rudder Internal"
+
                                         val isSystem = t.exists( _._2.isSystem == true) //isSystem should be consistant, but still
-                                        if(isSystem && mod == VersionAdded) {
-                                          logPure.info(s"Not auto-adding system techniques '${name.value}' in user library. You will need to add it explicitly.") *>
-                                          UIO.unit
+                                        val parentCat = if(isSystem && mod == VersionAdded) {
+                                          (ActiveTechniqueCategoryId("Rudder Internal"), "Active techniques used by Rudder")
                                         } else {
-                                          val referenceId = t.head._2.id
-
                                           val referenceCats = techniquesInfo.techniquesCategory(referenceId).getIdPathFromRoot.map( techniquesInfo.allCategories(_) )
-
-                                            //now, for each category in reference library, look if it exists in target library, and recurse on children
-                                            //tail of cats, because if root, we want an empty list to return immediatly in findCategory
-                                            val parentCat = findCategory(referenceCats.tail.map(x => CategoryInfo(x.id.name.value, x.name, x.description)), techLib)
-
-                                          logPure.info(s"Automatically adding technique '${name.value}' in category '${parentCat._2} (${parentCat._1.value})' of active techniques library") *>
-                                          rwActiveTechniqueRepo.addTechniqueInUserLibrary(parentCat._1, name, mods.keys.toSeq, modId, actor, reason).chainError(
-                                              s"Error when automatically activating technique '${name.value}'"
-                                          ).unit
+                                          //now, for each category in reference library, look if it exists in target library, and recurse on children
+                                          //tail of cats, because if root, we want an empty list to return immediatly in findCategory
+                                          findCategory(referenceCats.tail.map(x => CategoryInfo(x.id.name.value, x.name, x.description)), techLib)
                                         }
+                                        logPure.info(s"Automatically adding technique '${name.value}' in category '${parentCat._2} (${parentCat._1.value})' of active techniques library") *>
+                                        rwActiveTechniqueRepo.addTechniqueInUserLibrary(parentCat._1, name, mods.keys.toSeq, modId, actor, reason).chainError(
+                                            s"Error when automatically activating technique '${name.value}'"
+                                        ).unit
                                     }
 
                                 }

webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/services/servers/PolicyServerManagementService.scala

@@ -39,9 +39,11 @@ package com.normation.rudder.services.servers
 
 import com.normation.inventory.domain.NodeId
 import com.normation.rudder.domain.Constants
+
 import net.liftweb.common.Loggable
 import com.normation.eventlog.EventActor
 import com.normation.eventlog.ModificationId
+
 import ca.mrvisser.sealerate
 import cats.data.NonEmptyList
 import com.normation.cfclerk.domain.TechniqueName
@@ -50,13 +52,16 @@ import com.normation.ldap.sdk.LDAPConnectionProvider
 import com.normation.ldap.sdk.RwLDAPConnection
 import com.normation.rudder.domain.RudderDit
 import com.normation.rudder.domain.logger.ApplicationLogger
+
 import com.unboundid.ldap.sdk.DN
 import net.liftweb.common.Failure
 import net.liftweb.common.Full
+
 import zio._
 import zio.syntax._
 import com.normation.errors._
 import com.normation.eventlog.EventLogDetails
+import com.normation.ldap.sdk.LDAPEntry
 import com.normation.rudder.domain.RudderLDAPConstants
 import com.normation.rudder.domain.appconfig.RudderWebPropertyName
 import com.normation.rudder.domain.eventlog.AuthorizedNetworkModification
@@ -84,9 +89,11 @@ import com.normation.rudder.domain.queries.Query
 import com.normation.rudder.domain.queries.StringComparator
 import com.normation.rudder.repository.EventLogRepository
 import com.normation.rudder.rule.category.RuleCategoryId
+
 import com.normation.zio._
 import zio.json._
 import com.normation.rudder.services.servers.json._
+
 import com.softwaremill.quicklens._
 import net.liftweb.common.Box
 
@@ -316,16 +323,31 @@ class PolicyServerManagementServiceImpl(
    */
   val lock = Semaphore.make(1).runNow
 
+
+
+  private def getDefaultSettingEntryIfMissing = {
+    val entry = dit.APPCONFIG.propertyModel(RudderWebPropertyName(PROP_NAME))
+    entry.resetValuesTo(RudderLDAPConstants.A_PROPERTY_VALUE, """[{"id":"root","allowed-networks":[]}]""")
+    entry
+  }
+
   private def getLdap(con: RwLDAPConnection) = {
     for {
-      entry   <- con.get(dit.APPCONFIG.propertyDN(RudderWebPropertyName(PROP_NAME))).notOptional(s"LDAP setting entry ${PROP_NAME} was not found. It may be a bug, please report it.")
+      entry   <- con.get(dit.APPCONFIG.propertyDN(RudderWebPropertyName(PROP_NAME))).map {
+                   case Some(e) => e
+                   case None    => getDefaultSettingEntryIfMissing
+                 }
       json    <- entry(RudderLDAPConstants.A_PROPERTY_VALUE).notOptional(s"Value for policy servers is empty, while we should always have root server defined. It may be a bug, please report it.")
       servers <- json.fromJson[JPolicyServers].toIO
     } yield {
       (entry, servers.toPolicyServers())
     }
   }
 
+  /*
+   * Get policy servers. If the setting property is missing (for example in migration), returns just root with
+   * no allowed networks.
+   */
   override def getPolicyServers(): IOResult[PolicyServers] = {
     for {
       con     <- ldap
@@ -490,17 +512,16 @@ object PolicyServerConfigurationObjects {
   }
 
   val relayTechniques = List("server-common", "rudder-service-apache","rudder-service-relayd")
-  val rootTechniques = List("rudder-service-postgresql","rudder-service-slapd","rudder-service-webapp")
+  val rootTechniques = List("rudder-service-postgresql","rudder-service-slapd","rudder-service-webapp") ::: relayTechniques
 
-  def directiveCommonHasPolicyServer(nodeId: NodeId, hostname: String, policyServerId: NodeId) = {
+  def directiveCommonHasPolicyServer(nodeId: NodeId) = {
     TechniqueName("common") ->
       s"common-hasPolicyServer-${nodeId.value}".toDirective
         .modify(_.parameters).setTo(Map(
              "OWNER"              -> Seq("${rudder.node.admin}")
            , "UUID"               -> Seq("${rudder.node.id}")
-           , "POLICYSERVER"       -> Seq(hostname)
-           , "POLICYSERVER_ID"    -> Seq(policyServerId.value)
-           , "POLICYSERVER_ADMIN" -> Seq("root")
+           , "POLICYSERVER_ID"    -> Seq("${rudder.node.policyserver.id}")
+           , "POLICYSERVER_ADMIN" -> Seq("${rudder.node.policyserver.admin}")
          ))
         .modify(_.name).setTo(s"Common - ${nodeId.value}")
         .modify(_.shortDescription).setTo(s"Common policy for nodes with '${nodeId.value}' for policy server")
@@ -526,8 +547,8 @@ object PolicyServerConfigurationObjects {
     val objectType = ObjectCriterion("node", Seq(Criterion("policyServerId", StringComparator, None),Criterion("agentName", AgentComparator, None)))
     NodeGroup(
         NodeGroupId(s"hasPolicyServer-${nodeId.value}")
-      , s"All nodes managed by ${nodeId.value} policy server"
-      , s"All nodes known by Rudder directly connected to the ${nodeId.value} server. This group exists only as internal purpose and should not be used to configure Nodes."
+      , s"All nodes managed by '${nodeId.value}' policy server"
+      , s"All nodes known by Rudder directly connected to the '${nodeId.value}' server. This group exists only as internal purpose and should not be used to configure Nodes."
       , Nil
       , Some(
           Query(
@@ -575,18 +596,17 @@ object PolicyServerConfigurationObjects {
   }
 
   /*
-   * Get system configuration objects (directives, groups, rules) for policyServerId with given hostname.
-   * parentPolicyServerId is the policyServer'policyServer (and for root, it's itself)
+   * Get system configuration objects (directives, groups, rules) for policyServerId
    */
-  def getConfigurationObject(policyServerId: NodeId, hostname: String, parentPolicyServerId: NodeId): PolicyServerConfigurationObjects = {
+  def getConfigurationObject(policyServerId: NodeId): PolicyServerConfigurationObjects = {
     val techniques = if(policyServerId == Constants.ROOT_POLICY_SERVER_ID) {
       rootTechniques
     } else {
       relayTechniques
     }
 
     PolicyServerConfigurationObjects(
-        Map(directiveServerCommon(policyServerId), directiveCommonHasPolicyServer(policyServerId, hostname, parentPolicyServerId))
+        Map(directiveServerCommon(policyServerId), directiveCommonHasPolicyServer(policyServerId))
           ++ directiveServices(policyServerId).filter { case (t, d) => techniques.contains(t.value) }
       , groupHasPolicyServer(policyServerId) :: Nil
       , PolicyServerTarget(policyServerId) :: Nil

webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/repository/ldap/LoadDemoDataTest.scala

@@ -37,6 +37,8 @@
 
 package com.normation.rudder.repository.ldap
 
+import com.normation.ldap.ldif.DummyLDIFFileLogger
+
 import org.junit.runner._
 import org.specs2.mutable._
 import org.specs2.runner._
@@ -53,20 +55,6 @@ import com.normation.ldap.sdk.RwLDAPConnection
 @RunWith(classOf[JUnitRunner])
 class LoadDemoDataTest extends Specification {
 
-  val schemaLDIFs = (
-      "00-core" ::
-      "01-pwpolicy" ::
-      "04-rfc2307bis" ::
-      "05-rfc4876" ::
-      "099-0-inventory" ::
-      "099-1-rudder"  ::
-      Nil
-  ) map { name =>
-    // toURI is needed for https://issues.rudder.io/issues/19186
-    this.getClass.getClassLoader.getResource("ldap-data/schema/" + name + ".ldif").toURI.getPath
-  }
-
-  val baseDN = "cn=rudder-configuration"
   val bootstrapLDIFs = ("ldap/bootstrap.ldif" :: "ldap-data/inventory-sample-data.ldif" :: Nil) map { name =>
     // toURI is needed for https://issues.rudder.io/issues/19186
     this.getClass.getClassLoader.getResource(name).toURI.getPath
@@ -80,12 +68,7 @@ class LoadDemoDataTest extends Specification {
     i + x
   }
 
-  val ldap = InMemoryDsConnectionProvider[RwLDAPConnection with RoLDAPConnection](
-      baseDNs = baseDN :: Nil
-    , schemaLDIFPaths = schemaLDIFs
-    , bootstrapLDIFPaths = bootstrapLDIFs
-  )
-
+  val ldap = InitTestLDAPServer.newLdapConnectionProvider(bootstrapLDIFs)
 
   "The in memory LDAP directory" should {
 
@@ -95,3 +78,29 @@ class LoadDemoDataTest extends Specification {
     }
   }
 }
+
+object InitTestLDAPServer {
+  val schemaLDIFs = (
+      "00-core" ::
+      "01-pwpolicy" ::
+      "04-rfc2307bis" ::
+      "05-rfc4876" ::
+      "099-0-inventory" ::
+      "099-1-rudder"  ::
+      Nil
+  ) map { name =>
+    // toURI is needed for https://issues.rudder.io/issues/19186
+    this.getClass.getClassLoader.getResource("ldap-data/schema/" + name + ".ldif").toURI.getPath
+  }
+
+  val baseDN = "cn=rudder-configuration"
+
+  def newLdapConnectionProvider(fullLdifPaths: List[String]) = {
+    InMemoryDsConnectionProvider[RwLDAPConnection with RoLDAPConnection](
+        baseDNs = baseDN :: Nil
+      , schemaLDIFPaths = schemaLDIFs
+      , bootstrapLDIFPaths = fullLdifPaths
+      , ldifFileLogger = new DummyLDIFFileLogger()
+    )
+  }
+}