Merge branch 'userstory_16484/rl_integration_tests_v1' of github.com:…

…gpoblon/rudder into userstory_16484/rl_integration_tests_v1

relay/sources/Makefile

@@ -88,6 +88,7 @@ install: build
 	mkdir -p $(DESTDIR)/opt/rudder/share/man/man1/
 	mkdir -p $(DESTDIR)/opt/rudder/share/commands/
 	mkdir -p $(DESTDIR)/var/rudder/inventories/incoming
+	mkdir -p $(DESTDIR)/var/rudder/inventories/failed
 	mkdir -p $(DESTDIR)/var/rudder/inventories/accepted-nodes-updates
 	mkdir -p $(DESTDIR)/var/rudder/lib/ssl
 	mkdir -p $(DESTDIR)/var/rudder/lib/relay

relay/sources/rudder-pkg/lib/rudder-pkg/rudderPkg.py

@@ -320,3 +320,21 @@ def update():
             logging.debug("restoring %s from %s"%(utils.INDEX_PATH, utils.INDEX_PATH + ".bkp"))
             os.rename(utils.INDEX_PATH + ".bkp", utils.INDEX_PATH)
         utils.fail(e)
+
+"""
+    Upgrade all plugins install in their latest compatible version
+"""
+def upgrade_all(mode):
+    for p in utils.DB["plugins"].keys():
+        currentVersion = rpkg.PluginVersion(utils.DB["plugins"][p]["version"])
+        pkgs = plugin.Plugin(p)
+        pkgs.getAvailablePackages()
+        if mode == "nightly":
+            latestVersion = pkgs.getLatestCompatibleNightly().version
+        else:
+            latestVersion = pkgs.getLatestCompatibleRelease().version
+        if currentVersion < latestVersion:
+            print("The plugin %s is installed in version %s. The version %s %s is available, the plugin will be upgraded."%(p, currentVersion.pluginLongVersion, mode, latestVersion.pluginLongVersion))
+            package_install_latest([p], mode)
+        else:
+            print("No newer %s compatible versions found for the plugin %s"%(mode, p))

relay/sources/rudder-pkg/rudder-pkg

@@ -6,6 +6,7 @@ Rudder package manager
 Usage:
     rudder package [--debug] install-file <package.rpkg>...
     rudder package [--debug] install <package> [--version=<version>] [--nightly]
+    rudder package [--debug] upgrade-all [--nightly]
     rudder package [--debug] show <package> [--version=<version>] [--nightly]
     rudder package [--debug] remove <package>...
     rudder package [--debug] search <package>...
@@ -34,6 +35,9 @@ Commands:
     install
         download and install the latest released version of the plugin
 
+    upgrade-all
+        download and upgrade the installed plugins to their latest available version
+
     show
         show detailed informations on a given plugin
 
@@ -103,6 +107,12 @@ if __name__ == "__main__":
             rudderPkg.package_list_installed()
     elif args['update']:
         rudderPkg.update()
+    elif args['upgrade-all']:
+        if args['--nightly']:
+            rudderPkg.upgrade_all("nightly")
+        else:
+            rudderPkg.upgrade_all("release")
+
     elif args['licenses']:
         rudderPkg.update_licenses()
     elif args['search']:

relay/sources/rudder-server-relay-postinst

@@ -60,7 +60,7 @@ chmod 750 /opt/rudder/etc/relayd
 chmod 640 /opt/rudder/etc/relayd/main.conf
 chmod 640 /opt/rudder/etc/relayd/logging.conf
 
-for dir in /var/rudder/inventories/incoming /var/rudder/inventories/accepted-nodes-updates /var/rudder/reports/incoming /var/rudder/reports/failed
+for dir in /var/rudder/inventories/incoming /var/rudder/inventories/failed /var/rudder/inventories/accepted-nodes-updates /var/rudder/reports/incoming /var/rudder/reports/failed
 do
   chmod 770 ${dir}
   chown ${APACHE_USER}:rudder ${dir}

webapp/sources/ldap-inventory/inventory-api/src/main/scala/com/normation/inventory/services/provisioning/CheckInventoryDigest.scala

@@ -84,7 +84,7 @@ class ParseInventoryDigestFileV1 extends ParseInventoryDigestFile {
 
     for {
       loaded  <- Task.effect {
-                   import scala.collection.JavaConverters._
+                   import scala.jdk.CollectionConverters._
                    properties.load(is)
                    properties.asInstanceOf[java.util.Map[String, String]].asScala.toMap
                  } mapError  { ex =>

webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/FusionReportUnmarshaller.scala

@@ -436,7 +436,9 @@ class FusionReportUnmarshaller(
   private[this] def demux(report:InventoryReport) : InventoryReport = {
     //how can that be better ?
     var r = report
-    r = r.copy(machine = r.machine.copy( bios = report.machine.bios.groupBy(identity).map { case (x,seq) => x.copy(quantity = seq.size) }.toSeq ) )
+    r = r.copy(machine = r.machine.copy( bios = report.machine.bios.
+
+      groupBy(identity).map { case (x,seq) => x.copy(quantity = seq.size) }.toSeq ) )
     r = r.copy(machine = r.machine.copy( controllers = report.machine.controllers.groupBy(identity).map { case (x,seq) => x.copy(quantity = seq.size) }.toSeq ) )
     r = r.copy(machine = r.machine.copy( memories = demuxMemories(report.machine.memories) ) )
     r = r.copy(machine = r.machine.copy( ports = report.machine.ports.groupBy(identity).map { case (x,seq) => x.copy(quantity = seq.size) }.toSeq ) )

webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreUnmarshallCheckConsistency.scala

@@ -261,7 +261,7 @@ class PreUnmarshallCheckConsistency extends PreUnmarshall {
   private[this] def checkNodeUUID(uuid: String) : IOResult[String] = {
     uuid match {
       case uuidAuthCharRegex(x) => uuid.succeed
-      case _ => s"""The UUID '${uuid}' is not valid. It should be lesser than 50 chars and contains chars among the set [a-zA-Z0-9\-])""".inconsistency
+      case _ => s"""The UUID '${uuid}' is not valid. It should be lesser than 50 chars and contains chars among the set [a-zA-Z0-9\\-])""".inconsistency
     }
   }
 

webapp/sources/ldap-inventory/inventory-repository/pom.xml

@@ -67,7 +67,6 @@ along with Rudder.  If not, see <http://www.gnu.org/licenses/>.
     <dependency>
       <groupId>com.softwaremill.quicklens</groupId>
       <artifactId>quicklens_${scala-binary-version}</artifactId>
-      <version>1.4.11</version>
     </dependency>
   </dependencies>
 

webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/history/impl/FileHistoryLogRepository.scala

@@ -31,6 +31,8 @@ import org.joda.time.format.ISODateTimeFormat
 import zio._
 import zio.syntax._
 
+import scala.reflect.ClassTag
+
 /**
  * A trait that allows to write and read datas of type T
  * to/from files
@@ -66,7 +68,7 @@ trait IdToFilenameConverter[ID] {
  * Any datas type may be used, as long as they can be read/write from
  * files
  */
-class FileHistoryLogRepository[ID,T](
+class FileHistoryLogRepository[ID:ClassTag,T](
   val rootDir:String,
   val marshaller:FileMarshalling[T],
   val converter:IdToFilenameConverter[ID]
@@ -134,7 +136,7 @@ class FileHistoryLogRepository[ID,T](
                 InventoryError.System(s"Error when trying to get file names")
               )
      } yield {
-      res
+      res.toSeq
     }
   }
 

webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/InventoryHistoryLogRepository.scala

@@ -73,7 +73,7 @@ class FullInventoryFileMarshalling(
         case e : LDIFException => InventoryError.System(e.getMessage)
         case e : FileNotFoundException => InventoryError.System((s"History file '${in.getAbsolutePath}' was not found. It was likelly deleted"))
       }).flatMap { buf =>
-        fromLdapEntries.fromLdapEntries(buf.map(e => new LDAPEntry(e)))
+        fromLdapEntries.fromLdapEntries(buf.map(e => new LDAPEntry(e)).toSeq)
       }
     }
   }

webapp/sources/ldap-inventory/inventory-repository/src/main/scala/com/normation/inventory/ldap/core/ReadOnlySoftwareInventoryDAOImpl.scala

@@ -106,7 +106,7 @@ class ReadOnlySoftwareDAOImpl(
       softwareIds    =  softwareByNode.values.flatten.toSeq
       software       <- search(con, softwareIds)
     } yield {
-      softwareByNode.mapValues { ids => software.filter(s => ids.contains(s.id)) }
+      softwareByNode.view.mapValues { ids => software.filter(s => ids.contains(s.id)) }.toMap
     })
   }
 
@@ -139,7 +139,7 @@ class ReadOnlySoftwareDAOImpl(
       // fetch all nodes
       nodes           <- con.searchSub(nodeBaseSearch, IS(OC_NODE), A_NODE_UUID)
       mutSetSoftwares <- Ref.make[scala.collection.mutable.Set[SoftwareUuid]](scala.collection.mutable.Set())
-      _               <- ZIO.foreach(nodes.grouped(50).toIterable) { nodeEntries => // batch by 50 nodes to avoid destroying ram/ldap con
+      _               <- ZIO.foreach(nodes.grouped(50).to(Iterable)) { nodeEntries => // batch by 50 nodes to avoid destroying ram/ldap con
                            for {
                              nodeIds       <- ZIO.foreach(nodeEntries) { e => IOResult.effect(e(A_NODE_UUID).map(NodeId(_))).notOptional(s"Missing mandatory attribute '${A_NODE_UUID}'") }
                              t2            <- currentTimeMillis

webapp/sources/ldap-inventory/inventory-repository/src/test/scala/com/normation/history/impl/TestFileHistoryLogRepository.scala

@@ -65,7 +65,7 @@ class TestFileHistoryLogRepository {
   @Test def basicTest: Unit = {
     val id1 = "data1"
     assertEquals(Right(List()), repos.getIds.map(_.toList).runNow)
-    assertEquals(_:Left[RudderError, Any], repos.versions(id1).runNow)
+    assertEquals(Right(List()), repos.versions(id1).runNow)
 
     val data1 = "Some data 1\nwith multiple lines"
 

webapp/sources/ldap-inventory/inventory-repository/src/test/scala/com/normation/inventory/ldap/core/TestInventory.scala

@@ -271,6 +271,12 @@ class TestInventory extends Specification {
     }
   }
 
+  implicit class ForceGet[A, B](opt: Either[A, Option[B]]) {
+    def forceGet: Some[B]  = opt match {
+      case Right(Some(b)) => Some(b)
+      case _              => throw new Exception("in Test")
+    }
+  }
 
   "Saving, finding and moving node" should {
 
@@ -308,7 +314,7 @@ class TestInventory extends Specification {
         repo.save(full(n, m)).isOK
         and repo.move(n.main.id, PendingInventory, AcceptedInventory).isOK
         and {
-          val Some(FullInventory(node, machine)) = repo.get(n.main.id, AcceptedInventory).testRun.getOrElse(throw new Exception("in Test"))
+          val Some(FullInventory(node, machine)) = repo.get(n.main.id, AcceptedInventory).testRun.forceGet
 
           (
             machine === Some(m.copy(status = AcceptedInventory)) and
@@ -324,7 +330,7 @@ class TestInventory extends Specification {
       (
         repo.save(full(n, m)).isOK
         and {
-          val Some(FullInventory(node, machine)) = repo.get(n.main.id, PendingInventory).testRun.getOrElse(throw new Exception("in Test"))
+          val Some(FullInventory(node, machine)) = repo.get(n.main.id, PendingInventory).testRun.forceGet
 
           (
             node === n
@@ -340,7 +346,7 @@ class TestInventory extends Specification {
       (
         repo.save(full(n, m)).isOK
         and {
-          val Some(FullInventory(node, machine)) = repo.get(n.main.id, PendingInventory).testRun.getOrElse(throw new Exception("in Test"))
+          val Some(FullInventory(node, machine)) = repo.get(n.main.id, PendingInventory).testRun.forceGet
 
           (
             node === n
@@ -362,10 +368,10 @@ class TestInventory extends Specification {
         repo.save(FullInventory(n2,None)).isOK and repo.save(FullInventory(n3,None)).isOK
         and repo.move(n0.main.id, PendingInventory, AcceptedInventory).isOK
         and {
-          val Some(FullInventory(node0, m0)) = repo.get(n0.main.id, AcceptedInventory).testRun.getOrElse(throw new Exception("in Test"))
-          val Some(FullInventory(node1, m1)) = repo.get(n1.main.id, PendingInventory).testRun.getOrElse(throw new Exception("in Test"))
-          val Some(FullInventory(node2, m2)) = repo.get(n2.main.id, AcceptedInventory).testRun.getOrElse(throw new Exception("in Test"))
-          val Some(FullInventory(node3, m3)) = repo.get(n3.main.id, RemovedInventory).testRun.getOrElse(throw new Exception("in Test"))
+          val Some(FullInventory(node0, m0)) = repo.get(n0.main.id, AcceptedInventory).testRun.forceGet
+          val Some(FullInventory(node1, m1)) = repo.get(n1.main.id, PendingInventory).testRun.forceGet
+          val Some(FullInventory(node2, m2)) = repo.get(n2.main.id, AcceptedInventory).testRun.forceGet
+          val Some(FullInventory(node3, m3)) = repo.get(n3.main.id, RemovedInventory).testRun.forceGet
 
           //expected machine value
           val machine = m.copy(status = AcceptedInventory)
@@ -409,7 +415,7 @@ class TestInventory extends Specification {
       )
 
       repo.save(FullInventory(node, None)).isOK and {
-        val Some(FullInventory(n, m)) = repo.get(NodeId("windows 2012"), AcceptedInventory).testRun.getOrElse(throw new Exception("in Test"))
+        val Some(FullInventory(n, m)) = repo.get(NodeId("windows 2012"), AcceptedInventory).testRun.forceGet
         n === node
       }
 

webapp/sources/pom.xml

@@ -142,20 +142,18 @@ limitations under the License.
           <compilerPlugins>
             <compilerPlugin>
               <groupId>com.github.ghik</groupId>
-              <artifactId>silencer-plugin_${scala-binary-version}</artifactId>
+              <artifactId>silencer-plugin_${scala-version}</artifactId>
               <version>${silencer-lib-version}</version>
             </compilerPlugin>
           </compilerPlugins>
           <args>
             <!-- require java8, cf http://www.rudder-project.org/redmine/issues/9917 -->
             <arg>-target:jvm-1.8</arg>
-            <!-- needed for cats resolution of implicits -->
-            <arg>-Ypartial-unification</arg>
             <arg>-dependencyfile</arg>
             <arg>${basedir}/.scala_dependencies</arg>
             <!-- standard warning, most of the one from https://tpolecat.github.io/2017/04/25/scalac-flags.html -->
             <!-- corresponding scala line:
-              -Ypartial-unification  -language:existentials -language:higherKinds -language:implicitConversions
+              -language:existentials -language:higherKinds -language:implicitConversions
               -Xlint:_,-nullary-unit,-missing-interpolator -Yno-adapted-args -Ywarn-dead-code -Ywarn-extra-implicit -Ywarn-inaccessible
               -Ywarn-infer-any -Ywarn-nullary-override -Ywarn-numeric-widen -Ywarn-unused:imports -Ywarn-unused:locals -Ywarn-unused:privates
             -->
@@ -167,18 +165,19 @@ limitations under the License.
             <arg>-language:higherKinds</arg>         <!-- Allow higher-kinded types -->
             <arg>-language:implicitConversions</arg> <!-- Allow definition of implicit functions called views -->
             <arg>-Xcheckinit</arg>                   <!--  Wrap field accessors to throw an exception on uninitialized access. -->
-            <arg>-Xfuture</arg>                      <!-- Turn on future language features. -->
-            <arg>-Xlint:_,-nullary-unit,-missing-interpolator</arg> <!-- Xlint, minus nullary-units and missing interpolator (which brings false positive in Rudder) -->
-            <arg>-Yno-adapted-args</arg>             <!-- Do not adapt an argument list (either by inserting () or creating a tuple) to match the receiver. -->
+            <arg>-Xlint:_,-nonlocal-return,-nullary-unit,-missing-interpolator</arg> <!-- Xlint, minus non-local returns, nullary-units and missing interpolator (which brings false positive in Rudder) -->
             <arg>-Ywarn-dead-code</arg>              <!-- Warn when dead code is identified. -->
             <arg>-Ywarn-extra-implicit</arg>         <!-- Warn when more than one implicit parameter section is defined. -->
-            <arg>-Ywarn-inaccessible</arg>           <!-- Warn about inaccessible types in method signatures. -->
-            <arg>-Ywarn-infer-any</arg>              <!-- Warn when a type argument is inferred to be `Any`. -->
-            <arg>-Ywarn-nullary-override</arg>       <!-- Warn when non-nullary `def f()' overrides nullary `def f'. -->
             <arg>-Ywarn-numeric-widen</arg>          <!-- Warn when numerics are widened. -->
             <arg>-Ywarn-unused:imports</arg>         <!-- Warn if an import selector is not referenced. -->
             <arg>-Ywarn-unused:locals</arg>          <!-- Warn if a local definition is unused. -->
             <arg>-Ywarn-unused:privates</arg>        <!-- Warn if a private member is unused. -->
+            <arg>-Ywarn-unused:implicits</arg>       <!-- Warn if an implicit parameter is unused. -->
+            <arg>-Ywarn-unused:privates</arg>        <!-- Warn if a private member is unused. -->
+            <arg>-Ybackend-parallelism</arg><arg>8</arg>         <!-- Enable paralellisation — change to desired number! -->
+            <arg>-Ycache-plugin-class-loader:last-modified</arg> <!-- Enables caching of classloaders for compiler plugins -->
+            <arg>-Ycache-macro-class-loader:last-modified</arg>  <!-- and macro definitions. This can lead to performance improvements. -->
+            <arg>-P:silencer:checkUnused</arg>
           </args>
           <jvmArgs>
             <jvmArg>-Xmx${jvmArg-Xmx}</jvmArg>
@@ -350,11 +349,11 @@ limitations under the License.
     <rudder-major-version>6.0</rudder-major-version>
     <rudder-version>6.0.3-SNAPSHOT</rudder-version>
 
-    <scala-version>2.12.10</scala-version>
-    <scala-binary-version>2.12</scala-binary-version>
+    <scala-version>2.13.1</scala-version>
+    <scala-binary-version>2.13</scala-binary-version>
     <scala-parser-combinators-version>1.1.2</scala-parser-combinators-version>
     <scala-xml-version>1.2.0</scala-xml-version>
-    <lift-version>3.3.0</lift-version>
+    <lift-version>3.4.0</lift-version>
     <slf4j-version>1.7.28</slf4j-version>
     <logback-version>1.2.3</logback-version>
     <junit-version>4.12</junit-version>
@@ -368,12 +367,15 @@ limitations under the License.
     <cglib-version>3.3.0</cglib-version>
     <asm-version>5.2</asm-version>
     <bcpkix-jdk15on-version>1.63</bcpkix-jdk15on-version>
-    <silencer-lib-version>1.4.2</silencer-lib-version>
+    <silencer-lib-version>1.4.4</silencer-lib-version>
     <better-files-version>3.8.0</better-files-version>
     <sourcecode-version>0.1.7</sourcecode-version>
-    <quicklense-version>1.4.12</quicklense-version>
+    <quicklens-version>1.4.12</quicklens-version>
     <hikaricp-version>3.4.0</hikaricp-version>
+    <nuprocess-version>1.2.5</nuprocess-version>
     <postgresql-version>42.2.8</postgresql-version>
+    <json-path-version>2.4.0</json-path-version>
+    <scalaj-version>2.3.0</scalaj-version>
     <!--
       These one must be updated to work together
       We declare cats in "test" here, because it is not directly needed
@@ -382,9 +384,9 @@ limitations under the License.
     <monix-version>3.0.0</monix-version>
     <cats-version>2.0.0</cats-version>
     <specs2-version>4.7.0</specs2-version>
-    <doobie-version>0.8.2</doobie-version>
-    <fs2-version>1.0.4</fs2-version>
-    <http4s-version>0.19.0</http4s-version>
+    <doobie-version>0.8.8</doobie-version>
+    <fs2-version>2.1.0</fs2-version>
+    <http4s-version>0.21.0-M6</http4s-version>
     <shapeless-version>2.3.3</shapeless-version>
     <cats-effect-version>2.0.0</cats-effect-version>
     <dev-zio-version>1.0.0-RC16</dev-zio-version>
@@ -489,7 +491,7 @@ limitations under the License.
       <dependency>
         <groupId>com.softwaremill.quicklens</groupId>
         <artifactId>quicklens_${scala-binary-version}</artifactId>
-        <version>${quicklense-version}</version>
+        <version>${quicklens-version}</version>
       </dependency><!--
         lift-web
       -->
@@ -684,17 +686,18 @@ limitations under the License.
     <dependency>
       <groupId>com.github.pathikrit</groupId>
       <artifactId>better-files_${scala-binary-version}</artifactId>
+      <version>${better-files-version}</version>
     </dependency>
     <dependency>
       <groupId>com.github.ghik</groupId>
-      <artifactId>silencer-lib_${scala-binary-version}</artifactId>
+      <artifactId>silencer-lib_${scala-version}</artifactId>
       <version>${silencer-lib-version}</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>ca.mrvisser</groupId>
       <artifactId>sealerate_${scala-binary-version}</artifactId>
-      <version>0.0.5</version>
+      <version>0.0.6</version>
       <scope>provided</scope>
     </dependency>
     <!-- joda-time is used in all projects -->