Added checks for code formatting (#1138)

* Steps that check code formatting * sbt format Run `scalafix OrganizeImports` once. It's a one off change that will be checked by PR checks from now on. * update README * Nicer message
ShiftLeftSecurity · Mar 2, 2021 · 024f1cd · 024f1cd
1 parent 725b3e3
commit 024f1cd
Show file tree

Hide file tree

Showing 135 changed files with 241 additions and 274 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -22,8 +22,12 @@ jobs:
       run: sudo apt-get install -y --allow-downgrades sbt=1.3.12
     - name: Print sbt version
       run: sbt --version
-    - name: Run tests
-      run: sbt scalafmtCheck test
+    - name: Compile and run tests
+      run: sbt compile test
+    - name: Check formatting
+      run: sbt scalafmtCheck test:scalafmtCheck "scalafixAll --check OrganizeImports"
+    - run: echo "Previous step failed because code is not formatted. Run `sbt format`"
+      if: ${{ failure() }}
 
   test-fuzzypp-linux:
     runs-on: ubuntu-18.04

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -23,8 +23,12 @@ jobs:
       run: sudo apt-get install -y --allow-downgrades sbt=1.3.12
     - name: Print sbt version
       run: sbt --version
-    - name: Run tests
-      run: sbt scalafmtCheck test
+    - name: Compile and run tests
+      run: sbt compile test
+    - name: Check formatting
+      run: sbt scalafmtCheck test:scalafmtCheck "scalafixAll --check OrganizeImports"
+    - run: echo "Previous step failed because code is not formatted. Run `sbt format`"
+      if: ${{ failure() }}
 
   test-fuzzypp-linux:
     runs-on: ubuntu-18.04

diff --git a/README.md b/README.md
@@ -35,14 +35,10 @@ The build process has been verified on Linux and it should be possible
 to build on OS X and BSD systems as well. The build process requires
 the following prerequisites:
 
-* Java runtime 8
-  - Link: http://openjdk.java.net/install/
-* Scala build tool (sbt)
-  - Link: https://www.scala-sbt.org/
-* Git-lfs
-  - Link: https://git-lfs.github.com/
-* Protoc
-  - Link: https://github.com/protocolbuffers/protobuf/releases
+* [Java runtime 8](http://openjdk.java.net/install/)
+* [Scala build tool (SBT)](https://www.scala-sbt.org/)
+* [Git-lfs](https://git-lfs.github.com/)
+* [Protocol Buffer](https://github.com/protocolbuffers/protobuf/releases)
 
 Some binary files required for testing are managed through `git-lfs`. If you haven't cloned this repository yet, simply run `git lfs install`.
 If you have cloned it already, additionally run `git lfs pull` (from within the repository).
@@ -57,6 +53,15 @@ This command will install the following artifacts:
 
 * _codepropertygraph-protos-VERSION.jar_: Java bindings for Google's Protocol Buffer definitions
 
+# Code style
+
+Code style is automatically verified by external tools:
+
+* [scalafmt](https://github.com/scalameta/scalafmt)
+* [scalafix](https://github.com/scalacenter/scalafix)
+
+If your PR build fails code formatting check, simply run `sbt format` and submit the change along with the rest of the code. The commands runs necessary formatting in the right order.
+
 # Creating Protocol Buffer bindings for different languages
 
 The _codepropertygraph-VERSION.jar_ artifact contains a Protocol Buffer definition file _cpg.proto_ that you

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/Implicits.scala b/codepropertygraph/src/main/scala/io/shiftleft/Implicits.scala
@@ -1,7 +1,6 @@
 package io.shiftleft
 
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 
 object Implicits {
 

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/SerializedCpg.scala b/codepropertygraph/src/main/scala/io/shiftleft/SerializedCpg.scala
@@ -1,12 +1,12 @@
 package io.shiftleft
 
+import com.google.protobuf.GeneratedMessageV3
+
 import java.io.{File, IOException}
 import java.net.{URI, URISyntaxException}
 import java.nio.file.{FileSystem, FileSystems, Files}
 import java.util
 
-import com.google.protobuf.GeneratedMessageV3
-
 class SerializedCpg extends AutoCloseable {
 
   /**

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/Cpg.scala b/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/Cpg.scala
@@ -1,9 +1,8 @@
 package io.shiftleft.codepropertygraph
 
 import io.shiftleft.codepropertygraph.generated.{edges, nodes}
-import overflowdb.Graph
-import overflowdb.Config
 import overflowdb.traversal.help.TraversalHelp
+import overflowdb.{Config, Graph}
 
 object Cpg {
 

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/CpgLoader.scala b/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/CpgLoader.scala
@@ -3,8 +3,7 @@ package io.shiftleft.codepropertygraph.cpgloading
 import better.files.File
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.NodeKeys
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 import overflowdb.Graph
 
 import scala.util.Try

diff --git a/...ertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/CpgOverlayLoader.scala b/...ertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/CpgOverlayLoader.scala
@@ -1,7 +1,5 @@
 package io.shiftleft.codepropertygraph.cpgloading
 
-import java.io.IOException
-
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.StoredNode
 import io.shiftleft.passes.DiffGraph
@@ -10,6 +8,7 @@ import io.shiftleft.utils.StringInterner
 import org.slf4j.{Logger, LoggerFactory}
 import overflowdb._
 
+import java.io.IOException
 import scala.collection.mutable
 import scala.jdk.CollectionConverters._
 

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/NodeFilter.scala b/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/NodeFilter.scala
@@ -3,8 +3,8 @@ package io.shiftleft.codepropertygraph.cpgloading
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.Node
 import io.shiftleft.proto.cpg.Cpg.NodePropertyName
 
-import scala.jdk.CollectionConverters._
 import scala.collection.mutable
+import scala.jdk.CollectionConverters._
 
 /**
   * Removes duplicate nodes to avoid uniqueness restriction for parallel frontends.

diff --git a/...opertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ProtoCpgLoader.scala b/...opertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ProtoCpgLoader.scala
@@ -1,18 +1,15 @@
 package io.shiftleft.codepropertygraph.cpgloading
 
 import com.google.protobuf.GeneratedMessageV3
-
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.proto.cpg.Cpg.{CpgOverlay, CpgStruct, DiffGraph}
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.Edge
-import java.util.{Collection => JCollection, List => JList}
-import java.io.InputStream
-import java.nio.file.{Files, Path}
-
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import io.shiftleft.proto.cpg.Cpg.{CpgOverlay, CpgStruct, DiffGraph}
+import org.slf4j.{Logger, LoggerFactory}
 import overflowdb.Config
 
+import java.io.InputStream
+import java.nio.file.{Files, Path}
+import java.util.{Collection => JCollection, List => JList}
 import scala.collection.mutable.ArrayBuffer
 import scala.jdk.CollectionConverters._
 import scala.util.{Failure, Success, Try, Using}

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ProtoToCpg.scala b/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ProtoToCpg.scala
@@ -1,17 +1,14 @@
 package io.shiftleft.codepropertygraph.cpgloading
 
-import java.util.{NoSuchElementException, Collection => JCollection}
-
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.proto.cpg.Cpg.CpgStruct.{Edge, Node}
 import io.shiftleft.proto.cpg.Cpg.PropertyValue
 import io.shiftleft.proto.cpg.Cpg.PropertyValue.ValueCase._
 import io.shiftleft.utils.StringInterner
-
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 import overflowdb._
 
+import java.util.{Collection => JCollection, NoSuchElementException}
 import scala.jdk.CollectionConverters._
 
 object ProtoToCpg {

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ZipArchive.scala b/codepropertygraph/src/main/scala/io/shiftleft/codepropertygraph/cpgloading/ZipArchive.scala
@@ -4,7 +4,6 @@ import java.io.Closeable
 import java.nio.file.attribute.BasicFileAttributes
 import java.nio.file.{FileSystem, FileSystems, FileVisitResult, Files, Path, Paths, SimpleFileVisitor}
 import java.util.{Collection => JCollection}
-
 import scala.collection.mutable.ArrayBuffer
 import scala.jdk.CollectionConverters._
 

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/console/Query.scala b/codepropertygraph/src/main/scala/io/shiftleft/console/Query.scala
@@ -1,7 +1,7 @@
 package io.shiftleft.console
 
-import io.shiftleft.codepropertygraph.generated.nodes
 import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.codepropertygraph.generated.nodes
 import overflowdb.traversal.Traversal
 
 case class Query(name: String,

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/passes/CpgPass.scala b/codepropertygraph/src/main/scala/io/shiftleft/passes/CpgPass.scala
@@ -1,17 +1,12 @@
 package io.shiftleft.passes
 
 import com.google.protobuf.GeneratedMessageV3
+import io.shiftleft.SerializedCpg
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, StoredNode}
-import io.shiftleft.SerializedCpg
-import java.util
-import java.lang.{Long => JLong}
+import org.slf4j.{Logger, LoggerFactory}
 
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
-import overflowdb.Node
-import gnu.trove.map.hash.THashMap
-import scala.collection.mutable
+import java.lang.{Long => JLong}
 import scala.concurrent.duration.DurationLong
 
 /**

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/passes/DiffGraph.scala b/codepropertygraph/src/main/scala/io/shiftleft/passes/DiffGraph.scala
@@ -1,13 +1,12 @@
 package io.shiftleft.passes
 
-import java.security.MessageDigest
-
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.{CpgNode, NewNode, StoredNode}
 import io.shiftleft.proto.cpg.Cpg.{DiffGraph => DiffGraphProto}
 import overflowdb._
 import overflowdb.traversal._
 
+import java.security.MessageDigest
 import scala.collection.mutable
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 import scala.jdk.CollectionConverters._

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/passes/DiffGraphProtoSerializer.scala b/codepropertygraph/src/main/scala/io/shiftleft/passes/DiffGraphProtoSerializer.scala
@@ -1,31 +1,31 @@
 package io.shiftleft.passes
 
-import io.shiftleft.proto.cpg.Cpg.CpgStruct.Edge.EdgeType
-import io.shiftleft.proto.cpg.Cpg.CpgStruct.Node.NodeType
-import java.lang.{Long => JLong}
-
 import com.google.protobuf.ByteString
 import io.shiftleft.codepropertygraph.generated.nodes
-import io.shiftleft.codepropertygraph.generated.nodes.{CpgNode, NewNode, StoredNode}
+import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, StoredNode}
+import io.shiftleft.proto.cpg.Cpg.CpgStruct.Edge.EdgeType
+import io.shiftleft.proto.cpg.Cpg.CpgStruct.Node.NodeType
 import io.shiftleft.proto.cpg.Cpg.{
   AdditionalEdgeProperty,
   AdditionalNodeProperty,
   BoolList,
   ContainedRefs,
   CpgOverlay,
   CpgStruct,
+  DiffGraph => DiffGraphProto,
   DoubleList,
   EdgePropertyName,
   FloatList,
   IntList,
   LongList,
   NodePropertyName,
   PropertyValue,
-  StringList,
-  DiffGraph => DiffGraphProto
+  StringList
 }
 import overflowdb._
 
+import java.lang.{Long => JLong}
+
 object DiffGraphProtoSerializer {
   val nodePropertyNames: Set[String] = NodePropertyName.values().map { _.name() }.toSet
 }

diff --git a/codepropertygraph/src/main/scala/io/shiftleft/passes/ParallelCpgPass.scala b/codepropertygraph/src/main/scala/io/shiftleft/passes/ParallelCpgPass.scala
@@ -1,10 +1,9 @@
 package io.shiftleft.passes
-import java.util.concurrent.LinkedBlockingQueue
-
 import io.shiftleft.SerializedCpg
 import io.shiftleft.codepropertygraph.Cpg
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
+
+import java.util.concurrent.LinkedBlockingQueue
 
 abstract class ParallelCpgPass[T](cpg: Cpg, outName: String = "", keyPools: Option[Iterator[KeyPool]] = None)
     extends CpgPassBase {

diff --git a/...opertygraph/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/CpgLoaderTests.scala b/...opertygraph/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/CpgLoaderTests.scala
@@ -1,11 +1,11 @@
 package io.shiftleft.codepropertygraph.cpgloading
 
-import java.nio.file.FileSystemNotFoundException
-
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 import overflowdb.Config
 
+import java.nio.file.FileSystemNotFoundException
+
 /**
   * Specification of the CPGLoader. The loader allows CPGs to be loaded
   * from the CPG protobuf file format (based on Google protocol buffers).

diff --git a/...ropertygraph/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/DiffGraphTest.scala b/...ropertygraph/src/test/scala/io/shiftleft/codepropertygraph/cpgloading/DiffGraphTest.scala
@@ -1,13 +1,13 @@
 package io.shiftleft.codepropertygraph.cpgloading
 
 import io.shiftleft.OverflowDbTestInstance
-import overflowdb._
-import overflowdb.traversal._
 import io.shiftleft.codepropertygraph.generated._
 import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, StoredNode}
 import io.shiftleft.passes.{DiffGraph, IntervalKeyPool}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
+import overflowdb._
+import overflowdb.traversal._
 
 class DiffGraphTest extends AnyWordSpec with Matchers {
   "should be able to build an inverse DiffGraph" in {

diff --git a/codepropertygraph/src/test/scala/io/shiftleft/passes/CpgOverlayIntegrationTest.scala b/codepropertygraph/src/test/scala/io/shiftleft/passes/CpgOverlayIntegrationTest.scala
@@ -1,7 +1,5 @@
 package io.shiftleft.passes
 
-import java.util.Optional
-
 import io.shiftleft.OverflowDbTestInstance
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated._
@@ -10,6 +8,8 @@ import org.scalatest.wordspec.AnyWordSpec
 import overflowdb._
 import overflowdb.traversal._
 
+import java.util.Optional
+
 class CpgOverlayIntegrationTest extends AnyWordSpec with Matchers {
   val InitialNodeCode = "initialNode"
   val Pass1NewNodeCode = "pass1NewNodeCode"

diff --git a/codepropertygraph/src/test/scala/io/shiftleft/passes/CpgPassTests.scala b/codepropertygraph/src/test/scala/io/shiftleft/passes/CpgPassTests.scala
@@ -4,11 +4,11 @@ import better.files.File
 import io.shiftleft.SerializedCpg
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes
-
-import scala.jdk.CollectionConverters._
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 
+import scala.jdk.CollectionConverters._
+
 class CpgPassTests extends AnyWordSpec with Matchers {
 
   private object Fixture {

diff --git a/codepropertygraph/src/test/scala/io/shiftleft/x2cpg/X2CpgTests.scala b/codepropertygraph/src/test/scala/io/shiftleft/x2cpg/X2CpgTests.scala
@@ -1,8 +1,8 @@
 package io.shiftleft.x2cpg
 
+import better.files.File
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
-import better.files.File
 
 class X2CpgTests extends AnyWordSpec with Matchers {
 

diff --git a/console/src/main/scala/io/shiftleft/console/BridgeBase.scala b/console/src/main/scala/io/shiftleft/console/BridgeBase.scala
@@ -1,11 +1,10 @@
 package io.shiftleft.console
 
-import ammonite.ops.pwd
-import ammonite.ops.Path
+import ammonite.ops.{Path, pwd}
 import ammonite.util.{Colors, Res}
 import better.files._
-import io.shiftleft.console.embammonite.EmbeddedAmmonite
 import io.shiftleft.console.cpgqlserver.CPGQLServer
+import io.shiftleft.console.embammonite.EmbeddedAmmonite
 
 import java.io.{FileOutputStream, PrintStream}
 

diff --git a/console/src/main/scala/io/shiftleft/console/Help.scala b/console/src/main/scala/io/shiftleft/console/Help.scala
@@ -1,7 +1,7 @@
 package io.shiftleft.console
 
-import overflowdb.traversal.help.{Doc, Table}
 import org.apache.commons.lang.WordUtils
+import overflowdb.traversal.help.{Doc, Table}
 
 import scala.reflect.runtime.universe.{TypeTag, typeOf}
 

diff --git a/console/src/main/scala/io/shiftleft/console/PluginManager.scala b/console/src/main/scala/io/shiftleft/console/PluginManager.scala
@@ -1,6 +1,6 @@
 package io.shiftleft.console
-import better.files.File
 import better.files.Dsl._
+import better.files.File
 import better.files.File.apply
 
 import java.nio.file.Path

diff --git a/console/src/main/scala/io/shiftleft/console/QueryDatabase.scala b/console/src/main/scala/io/shiftleft/console/QueryDatabase.scala
@@ -3,6 +3,7 @@ package io.shiftleft.console
 import org.reflections8.Reflections
 import org.reflections8.util.{ClasspathHelper, ConfigurationBuilder}
 import org.slf4j.{Logger, LoggerFactory}
+
 import scala.annotation.{StaticAnnotation, unused}
 import scala.jdk.CollectionConverters._
 import scala.reflect.runtime.universe._