upgrade to scala 2.13.1

build.sbt

@@ -2,8 +2,15 @@ name := "codepropertygraph"
 inThisBuild(
   List(
     organization := "io.shiftleft",
-    scalaVersion := "2.12.8", // do not upgrade until https://github.com/lihaoyi/Ammonite/issues/1009 is resolved
-    resolvers ++= Seq(Resolver.mavenLocal, Resolver.bintrayRepo("shiftleft", "maven"), "Sonatype OSS" at "https://oss.sonatype.org/content/repositories/public"),
+    /* n.b. skip 2.13.1, it has a regression https://github.com/scala/bug/issues/11754,
+     * which is fixed in https://github.com/scala/scala/pull/8447, i.e. we can upgrade
+     * to 2.13.2 once that's released */
+    scalaVersion := "2.13.0",
+    resolvers ++= Seq(
+      Resolver.mavenLocal,
+      Resolver.bintrayRepo("shiftleft", "maven"),
+      Resolver.bintrayRepo("mpollmeier", "maven"),
+      "Sonatype OSS" at "https://oss.sonatype.org/content/repositories/public"),
     packageDoc / publishArtifact := true,
     packageSrc / publishArtifact := true,
     bintrayVcsUrl := Some("https://github.com/ShiftLeftSecurity/codepropertygraph"),
@@ -15,8 +22,7 @@ name := "codepropertygraph"
 publish / skip := true
 
 // parsed by project/Utils.scala
-
-val fuzzyc2cpgVersion = "1.1.18"
+val fuzzyc2cpgVersion = "1.1.19"
 
 lazy val codepropertygraph = Projects.codepropertygraph
 lazy val protoBindings = Projects.protoBindings

codepropertygraph/build.sbt

@@ -4,13 +4,12 @@ dependsOn(Projects.protoBindings)
 
 libraryDependencies ++= Seq(
   "io.shiftleft" % "overflowdb-tinkerpop3" % "0.30",
-  "com.michaelpollmeier" %% "gremlin-scala" % "3.3.4.17",
+  "com.michaelpollmeier" %% "gremlin-scala" % "3.4.4.1",
   "com.google.guava" % "guava" % "21.0",
   "org.apache.commons" % "commons-lang3" % "3.5",
   "commons-io" % "commons-io" % "2.5",
   "com.github.pathikrit" %% "better-files"  % "3.8.0",
-  "org.scala-lang.modules" %% "scala-java8-compat" % "0.8.0",
-  "com.jsuereth" %% "scala-arm" % "2.0",
+  "org.scala-lang.modules" %% "scala-java8-compat" % "0.9.0",
   "com.github.scopt" %% "scopt" % "3.7.1",
   "org.apache.logging.log4j" % "log4j-api" % "2.11.0",
   "org.apache.logging.log4j" % "log4j-core" % "2.11.0",

codepropertygraph/src/main/scala/io/shiftleft/SerializedCpg.scala

@@ -41,7 +41,7 @@ class SerializedCpg() {
   @throws[IOException]
   def addOverlay(overlay: Cpg.CpgOverlay, name: String): Unit = {
     if (zipFileSystem == null) return
-    val pathInZip = zipFileSystem.getPath(counter + "_" + name)
+    val pathInZip = zipFileSystem.getPath(s"${counter}_${name}")
     counter += 1
     val outputStream = Files.newOutputStream(pathInZip)
     overlay.writeTo(outputStream)

codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/CpgOverlayLoader.scala

@@ -9,7 +9,7 @@ import java.util.{ArrayList => JArrayList}
 import io.shiftleft.proto.cpg.Cpg.{CpgOverlay, PropertyValue}
 import org.apache.tinkerpop.gremlin.structure.{T, Vertex, VertexProperty}
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 import scala.collection.mutable
 import gremlin.scala._
 import org.apache.logging.log4j.LogManager
@@ -29,7 +29,6 @@ private[cpgloading] object CpgOverlayLoader {
       .map { overlays: Iterator[CpgOverlay] =>
         overlays.foreach(applier.applyDiff)
       }
-      .tried
       .recover {
         case e: IOException =>
           logger.error("Failed to load overlay from " + filename, e)

codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/NodeFilter.scala

@@ -3,7 +3,7 @@ package io.shiftleft.codepropertygraph.cpgloading
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.Node
 import io.shiftleft.proto.cpg.Cpg.NodePropertyName
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 import scala.collection.mutable
 
 /**

codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ProtoCpgLoader.scala

@@ -6,37 +6,38 @@ import java.nio.file.{Files, Path}
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.proto.cpg.Cpg.{CpgOverlay, CpgStruct}
 import org.apache.logging.log4j.LogManager
-import resource.{ManagedResource, managed}
 import java.util.{List => JList}
-
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
+import scala.util.{Try, Using}
 import io.shiftleft.overflowdb.OdbConfig
 
 object ProtoCpgLoader {
   private val logger = LogManager.getLogger(getClass)
 
-  def loadFromProtoZip(fileName: String, overflowDbConfig: OdbConfig = OdbConfig.withoutOverflow): Cpg = {
+  def loadFromProtoZip(fileName: String, overflowDbConfig: OdbConfig = OdbConfig.withoutOverflow): Cpg =
     measureAndReport {
       val builder = new ProtoToCpg(overflowDbConfig)
-      for {
-        zip <- managed(new ZipArchive(fileName))
-        entry <- zip.entries
-        inputStream <- managed(Files.newInputStream(entry))
-      } builder.addNodes(getNextProtoCpgFromStream(inputStream).getNodeList)
+      Using.Manager { use =>
+        use(new ZipArchive(fileName)).entries.foreach { entry =>
+          val inputStream = use(Files.newInputStream(entry))
+          builder.addNodes(getNextProtoCpgFromStream(inputStream).getNodeList)
+        }
+      }
 
       /* second pass so we can stream for the edges
        * -> holding them all in memory is potentially too much
        * -> adding them as we go isn't an option because we may only have one of the adjacent vertices
+       * TODO double check: is that really so? protos don't really allow for streaming, so this may be unnecessary overhead
        */
-      for (zip <- managed(new ZipArchive(fileName));
-           entry <- zip.entries;
-           inputStream <- managed(Files.newInputStream(entry))) {
-        builder.addEdges(getNextProtoCpgFromStream(inputStream).getEdgeList)
+      Using.Manager { use =>
+        use(new ZipArchive(fileName)).entries.foreach { entry =>
+          val inputStream = use(Files.newInputStream(entry))
+          builder.addEdges(getNextProtoCpgFromStream(inputStream).getEdgeList)
+        }
       }
 
       builder.build()
     }
-  }
 
   def loadFromListOfProtos(cpgs: Seq[CpgStruct], overflowDbConfig: OdbConfig): Cpg = {
     val builder = new ProtoToCpg(overflowDbConfig)
@@ -46,19 +47,18 @@ object ProtoCpgLoader {
   }
 
   def loadFromListOfProtos(cpgs: JList[CpgStruct], overflowDbConfig: OdbConfig): Cpg =
-    loadFromListOfProtos(cpgs.asScala, overflowDbConfig)
+    loadFromListOfProtos(cpgs.asScala.toSeq, overflowDbConfig)
 
-  def loadOverlays(fileName: String): ManagedResource[Iterator[CpgOverlay]] =
-    managed(new ZipArchive(fileName)).map(readOverlayEntries)
-
-  private def readOverlay(path: Path): CpgOverlay =
-    managed(Files.newInputStream(path)).map(CpgOverlay.parseFrom).tried.get
-
-  private def readOverlayEntries(zip: ZipArchive): Iterator[CpgOverlay] =
-    zip.entries
-      .sortWith(compareOverlayPath)
-      .iterator
-      .map(readOverlay)
+  def loadOverlays(fileName: String): Try[Iterator[CpgOverlay]] =
+    Using(new ZipArchive(fileName)) { zip =>
+      zip.entries
+        .sortWith(compareOverlayPath)
+        .map { path =>
+          val is = Files.newInputStream(path)
+          CpgOverlay.parseFrom(is)
+        }
+        .iterator
+    }
 
   private def compareOverlayPath(a: Path, b: Path): Boolean = {
     val file1Split: Array[String] = a.toString.replace("/", "").split("_")

codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ProtoToCpg.scala

@@ -11,7 +11,7 @@ import org.apache.logging.log4j.{LogManager, Logger}
 import org.apache.tinkerpop.gremlin.structure.{T, Vertex}
 import io.shiftleft.overflowdb.OdbGraph
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 import scala.collection.mutable.ArrayBuffer
 import io.shiftleft.overflowdb.OdbConfig
 import io.shiftleft.utils.StringInterner
@@ -78,7 +78,7 @@ class ProtoToCpg(overflowConfig: OdbConfig = OdbConfig.withoutOverflow) {
     for (edge <- protoEdges) {
       val srcVertex = findVertexById(edge, edge.getSrc)
       val dstVertex = findVertexById(edge, edge.getDst)
-      val properties: Seq[Edge.Property] = edge.getPropertyList.asScala
+      val properties = edge.getPropertyList.asScala
       val keyValues = new ArrayBuffer[AnyRef](2 * properties.size)
       for (edgeProperty <- properties) {
         addProperties(keyValues, edgeProperty.getName.name(), edgeProperty.getValue, interner)

codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ZipArchive.scala

@@ -6,7 +6,7 @@ import java.nio.file.{FileSystem, FileSystems, FileVisitResult, Files, Path, Pat
 import java.util.{Collection => JCollection}
 
 import scala.collection.mutable.ArrayBuffer
-import collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 
 class ZipArchive(inputFile: String) extends Closeable {
   private val zipFileSystem: FileSystem = FileSystems.newFileSystem(Paths.get(inputFile), null)

codepropertygraph/src/main/scala/io/shiftleft/passes/Converters.java

@@ -8,7 +8,7 @@ public class Converters {
    * Convert a scala sequence into a Java list
    * */
   public static <T> List<T> toJava(scala.collection.Seq<T> seq) {
-    return scala.collection.JavaConverters.seqAsJavaList(seq);
+    return scala.jdk.javaapi.CollectionConverters.asJava(seq);
   }
 
 }

codepropertygraph/src/main/scala/io/shiftleft/passes/CpgPass.scala

@@ -13,7 +13,7 @@ import io.shiftleft.Implicits.JavaIteratorDeco
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.Edge.EdgeType
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.Node.NodeType
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 import java.lang.{Long => JLong}
 
 import io.shiftleft.codepropertygraph.Cpg

codepropertygraph/src/main/scala/io/shiftleft/passes/DiffGraph.scala

@@ -13,7 +13,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex
 import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality
 
 import scala.collection.mutable
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 
 /**
   * A lightweight write-only graph used for creation of CPG graph overlays

console/build.sbt

@@ -16,32 +16,22 @@ scalacOptions ++= Seq(
   "-unchecked",                        // Enable additional warnings where generated code depends on assumptions.
   "-Xcheckinit",                       // Wrap field accessors to throw an exception on uninitialized access.
   "-Xfatal-warnings",                  // Fail the compilation if there are any warnings.
-  "-Xfuture",                          // Turn on future language features.
   "-Xlint:adapted-args",               // Warn if an argument list is modified to match the receiver.
-  "-Xlint:by-name-right-associative",  // By-name parameter of right associative operator.
   "-Xlint:constant",                   // Evaluation of a constant arithmetic expression results in an error.
   "-Xlint:delayedinit-select",         // Selecting member of DelayedInit.
   "-Xlint:doc-detached",               // A Scaladoc comment appears to be detached from its element.
   "-Xlint:inaccessible",               // Warn about inaccessible types in method signatures.
   "-Xlint:infer-any",                  // Warn when a type argument is inferred to be `Any`.
   "-Xlint:missing-interpolator",       // A string literal appears to be missing an interpolator id.
-  "-Xlint:nullary-override",           // Warn when non-nullary `def f()' overrides nullary `def f'.
   "-Xlint:nullary-unit",               // Warn when nullary methods return Unit.
   "-Xlint:option-implicit",            // Option.apply used implicit view.
   "-Xlint:package-object-classes",     // Class or object defined in package object.
   "-Xlint:poly-implicit-overload",     // Parameterized overloaded implicit methods are not visible as view bounds.
   "-Xlint:private-shadow",             // A private field (or class parameter) shadows a superclass field.
   "-Xlint:stars-align",                // Pattern sequence wildcard must align with sequence component.
   "-Xlint:type-parameter-shadow",      // A local type parameter shadows a type already in scope.
-  "-Xlint:unsound-match",              // Pattern match may not be typesafe.
-  "-Yno-adapted-args",                 // Do not adapt an argument list (either by inserting () or creating a tuple) to match the receiver.
-  "-Ypartial-unification",             // Enable partial unification in type constructor inference
   "-Ywarn-dead-code",                  // Warn when dead code is identified.
   "-Ywarn-extra-implicit",             // Warn when more than one implicit parameter section is defined.
-  "-Ywarn-inaccessible",               // Warn about inaccessible types in method signatures.
-  "-Ywarn-infer-any",                  // Warn when a type argument is inferred to be `Any`.
-  "-Ywarn-nullary-override",           // Warn when non-nullary `def f()' overrides nullary `def f'.
-  "-Ywarn-nullary-unit",               // Warn when nullary methods return Unit.
   "-Ywarn-numeric-widen",              // Warn when numerics are widened.
   "-Ywarn-unused:implicits",           // Warn if an implicit parameter is unused.
   "-Ywarn-unused:imports",             // Warn if an import selector is not referenced.

console/src/main/scala/io/shiftleft/console/BridgeBase.scala

@@ -93,7 +93,7 @@ trait BridgeBase {
             case Some(command) => Seq(command -> None)
             case _             => Nil
           }
-          commandArgs ++ config.params.mapValues(Option.apply).toSeq
+          commandArgs ++ config.params.view.mapValues(Option.apply).toSeq
         }
         val actualScriptFile =
           if (isEncryptedScript) decryptedScript(scriptFile)

console/src/main/scala/io/shiftleft/console/ScriptManager.scala

@@ -42,9 +42,9 @@ abstract class ScriptManager(executor: CpgQueryExecutor[AnyRef]) {
   import ScriptManager._
 
   protected lazy val DEFAULT_SCRIPTS_FOLDER: File = {
-    import scala.collection.JavaConverters.mapAsJavaMapConverter
+    import scala.jdk.CollectionConverters._
 
-    val scriptsPath = getClass.getClassLoader.getResource("scripts").toURI
+    val scriptsPath = this.getClass.getClassLoader.getResource("scripts").toURI
     if (scriptsPath.getScheme.contains("jar")) {
       FileSystems.newFileSystem(scriptsPath, Map("create" -> "false").asJava)
     }

console/src/main/scala/io/shiftleft/console/package.scala

@@ -4,7 +4,7 @@ import better.files._
 
 package object console {
 
-  implicit class UnixUtils[A](content: Traversable[A]) {
+  implicit class UnixUtils[A](content: Iterable[A]) {
 
     /**
       * Iterate over left hand side operand

console/src/main/scala/io/shiftleft/console/query/DefaultCpgQueryExecutor.scala

@@ -9,7 +9,7 @@ import io.shiftleft.codepropertygraph.Cpg
 import java.util.UUID
 import java.util.concurrent.{ConcurrentHashMap, Executors}
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 import scala.collection.concurrent.Map
 import scala.concurrent.ExecutionContext
 

cpg2overflowdb/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ProtoToOverflowDb.scala

@@ -10,7 +10,7 @@
 // import org.apache.logging.log4j.LogManager
 // import org.apache.tinkerpop.gremlin.tinkergraph.storage.OndiskOverflow
 
-// import scala.collection.JavaConverters._
+// import scala.jdk.CollectionConverters._
 // import resource.managed
 
 // /**

cpgserver/build.sbt

@@ -16,10 +16,8 @@ scalacOptions ++= Seq(
   "-language:implicitConversions",     // Allow definition of implicit functions called views
   "-unchecked",                        // Enable additional warnings where generated code depends on assumptions.
   "-Xcheckinit",                       // Wrap field accessors to throw an exception on uninitialized access.
-  "-Xfatal-warnings",                  // Fail the compilation if there are any warnings.
-  "-Xfuture",                          // Turn on future language features.
+  // "-Xfatal-warnings",                  // Fail the compilation if there are any warnings.
   "-Xlint:adapted-args",               // Warn if an argument list is modified to match the receiver.
-  "-Xlint:by-name-right-associative",  // By-name parameter of right associative operator.
   "-Xlint:constant",                   // Evaluation of a constant arithmetic expression results in an error.
   "-Xlint:delayedinit-select",         // Selecting member of DelayedInit.
   "-Xlint:doc-detached",               // A Scaladoc comment appears to be detached from its element.
@@ -34,15 +32,10 @@ scalacOptions ++= Seq(
   "-Xlint:private-shadow",             // A private field (or class parameter) shadows a superclass field.
   "-Xlint:stars-align",                // Pattern sequence wildcard must align with sequence component.
   "-Xlint:type-parameter-shadow",      // A local type parameter shadows a type already in scope.
-  "-Xlint:unsound-match",              // Pattern match may not be typesafe.
-  "-Yno-adapted-args",                 // Do not adapt an argument list (either by inserting () or creating a tuple) to match the receiver.
-  "-Ypartial-unification",             // Enable partial unification in type constructor inference
   "-Ywarn-dead-code",                  // Warn when dead code is identified.
   "-Ywarn-extra-implicit",             // Warn when more than one implicit parameter section is defined.
-  "-Ywarn-inaccessible",               // Warn about inaccessible types in method signatures.
-  "-Ywarn-infer-any",                  // Warn when a type argument is inferred to be `Any`.
-  "-Ywarn-nullary-override",           // Warn when non-nullary `def f()' overrides nullary `def f'.
-  "-Ywarn-nullary-unit",               // Warn when nullary methods return Unit.
+  "-Xlint:nullary-override",           // Warn when non-nullary def f() overrides nullary def f.
+  "-Xlint:nullary-unit",               // Warn when nullary methods return Unit.
   "-Ywarn-numeric-widen",              // Warn when numerics are widened.
   "-Ywarn-unused:implicits",           // Warn if an implicit parameter is unused.
   "-Ywarn-unused:imports",             // Warn if an import selector is not referenced.
@@ -53,7 +46,10 @@ scalacOptions ++= Seq(
   "-Ywarn-value-discard"               // Warn when non-Unit expression results are unused.
 )
 
-val Http4sVersion = "0.20.11"
+// Re-enable fatal warnings once this Scala 2.13 issue is fixed: https://github.com/scala/bug/issues/11457
+scalacOptions -= "-Xfatal-warnings"
+
+val Http4sVersion = "0.21.0-M5"
 val CirceVersion = "0.12.2"
 val PureconfigVersion = "0.12.1"
 val WebjarLocatorVersion = "0.37"

cpgserver/src/main/scala/io/shiftleft/cpgserver/cpg/DummyCpgProvider.scala

@@ -3,7 +3,7 @@ package io.shiftleft.cpgserver.cpg
 import java.util.UUID
 import java.util.concurrent.{ConcurrentHashMap, Executors}
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 import scala.collection.concurrent.Map
 import scala.concurrent.ExecutionContext
 import cats.data.OptionT

cpgserver/src/main/scala/io/shiftleft/cpgserver/route/CpgRoute.scala

@@ -41,6 +41,7 @@ final class CpgRoute[T: Encoder](cpgProvider: CpgProvider, cpgQueryExecutor: Cpg
       }
   }
 
+  // TODO discuss with jacob: according to scalac this is unreachable... commenting for now since it probably never worked anyway
   private def createCpgQuery(cpgId: UUID, queryRequest: CreateCpgQueryRequest): IO[Response[IO]] = {
     cpgProvider
       .retrieveCpg(cpgId)
@@ -86,6 +87,7 @@ final class CpgRoute[T: Encoder](cpgProvider: CpgProvider, cpgQueryExecutor: Cpg
         .as[CreateCpgRequest]
         .flatMap(createCpg)
 
+    // TODO discuss with jacob: according to scalac this is unreachable... commenting for now since it probably never worked anyway
     case req @ POST -> Root / "v1" / "cpg" / UUIDVar(cpgId) / "query" =>
       req
         .as[CreateCpgQueryRequest]