Evaluator module

cli/build.gradle

@@ -17,16 +17,14 @@ repositories {
 dependencies {
     compile project(':analyzer')
     compile project(':downloader')
+    compile project(':evaluator')
     compile project(':model')
     compile project(':reporter')
     compile project(':scanner')
     compile project(':utils')
 
     compile "com.beust:jcommander:$jcommanderVersion"
 
-    compile 'org.jetbrains.kotlin:kotlin-compiler-embeddable'
-    compile 'org.jetbrains.kotlin:kotlin-script-runtime'
-    compile 'org.jetbrains.kotlin:kotlin-script-util'
     compile 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
     compile 'org.jetbrains.kotlin:kotlin-reflect'
 

cli/src/main/kotlin/commands/EvaluatorCommand.kt

@@ -19,15 +19,12 @@
 
 package com.here.ort.commands
 
-import ch.frankel.slf4k.*
-
 import com.beust.jcommander.JCommander
 import com.beust.jcommander.Parameter
 import com.beust.jcommander.Parameters
 
 import com.here.ort.CommandWithHelp
-import com.here.ort.model.Error
-import com.here.ort.model.EvaluatorRun
+import com.here.ort.evaluator.Evaluator
 import com.here.ort.model.OrtResult
 import com.here.ort.model.OutputFormat
 import com.here.ort.model.readValue
@@ -38,12 +35,6 @@ import com.here.ort.utils.safeMkdirs
 
 import java.io.File
 
-import javax.script.Compilable
-import javax.script.ScriptEngineManager
-import javax.script.ScriptException
-
-import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
-
 @Parameters(commandNames = ["evaluate"], commandDescription = "Evaluate rules on ORT result files.")
 object EvaluatorCommand : CommandWithHelp() {
     @Parameter(description = "The ORT result file to read as input.",
@@ -78,59 +69,23 @@ object EvaluatorCommand : CommandWithHelp() {
             order = PARAMETER_ORDER_OPTIONAL)
     private var syntaxCheck = false
 
-    private val engine = ScriptEngineManager().getEngineByExtension("kts")
-
     override fun runCommand(jc: JCommander): Int {
         require((rulesFile == null) != (rulesResource == null)) {
             "Either '--rules-file' or '--rules-resource' must be specified."
         }
 
-        // This is required to avoid
-        //
-        //     WARN: Failed to initialize native filesystem for Windows
-        //     java.lang.RuntimeException: Could not find installation home path. Please make sure bin/idea.properties
-        //     is present in the installation directory.
-        //
-        // on Windows for some reason.
-        setIdeaIoUseFallback()
-
         val ortResultInput = ortFile.readValue<OrtResult>()
-        engine.put("ortResult", ortResultInput)
-
-        val preface = """
-            import com.here.ort.model.Error
-            import com.here.ort.model.OrtResult
-
-            val ortResult = bindings["ortResult"] as OrtResult
-            val evalErrors = mutableListOf<Error>()
 
-        """.trimIndent()
+        val script = rulesFile?.readText() ?: javaClass.getResource(rulesResource).readText()
 
-        val postface = """
-
-            evalErrors
-        """.trimIndent()
-
-        val script = preface + (rulesFile?.readText() ?: javaClass.getResource(rulesResource).readText()) + postface
+        val evaluator = Evaluator()
 
         if (syntaxCheck) {
-            require(engine is Compilable) {
-                "The scripting engine does not support compilation."
-            }
-
-            val canCompile = try {
-                engine.compile(script)
-                true
-            } catch (e: ScriptException) {
-                log.error { e.message?.let { it } ?: "No error message available." }
-                false
-            }
-
-            return if (canCompile) 0 else 1
+            return if (evaluator.checkSyntax(ortResultInput, script)) 0 else 1
         }
 
         @Suppress("UNCHECKED_CAST")
-        val evalErrors = engine.eval(script) as List<Error>
+        val evaluatorRun = evaluator.evaluate(ortResultInput, script)
 
         outputDir?.let { dir ->
             require(!dir.exists()) {
@@ -140,7 +95,7 @@ object EvaluatorCommand : CommandWithHelp() {
             dir.safeMkdirs()
 
             // Note: This overwrites any existing EvaluatorRun from the input file.
-            val ortResultOutput = ortResultInput.copy(evaluator = EvaluatorRun(evalErrors))
+            val ortResultOutput = ortResultInput.copy(evaluator = evaluatorRun)
 
             outputFormats.distinct().forEach { format ->
                 val evaluationResultFile = File(dir, "evaluation-result.${format.fileExtension}")
@@ -149,12 +104,12 @@ object EvaluatorCommand : CommandWithHelp() {
             }
         }
 
-        return if (evalErrors.isEmpty()) 0 else 1.also {
-            if (log.isErrorEnabled) {
-                evalErrors.forEach { error ->
-                    log.error(error.toString())
-                }
+        if (log.isErrorEnabled) {
+            evaluatorRun.errors.forEach { error ->
+                log.error(error.toString())
             }
         }
+
+        return if (evaluatorRun.errors.isEmpty()) 0 else 1
     }
 }

evaluator/build.gradle

@@ -0,0 +1,12 @@
+// Apply core plugins.
+apply plugin: 'java-library'
+
+dependencies {
+    api project(':model')
+
+    implementation project(':utils')
+
+    implementation 'org.jetbrains.kotlin:kotlin-compiler-embeddable'
+    implementation 'org.jetbrains.kotlin:kotlin-script-runtime'
+    implementation 'org.jetbrains.kotlin:kotlin-script-util'
+}

evaluator/src/main/kotlin/Evaluator.kt

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2017-2018 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package com.here.ort.evaluator
+
+import ch.frankel.slf4k.*
+
+import com.here.ort.model.Error
+import com.here.ort.model.EvaluatorRun
+import com.here.ort.model.OrtResult
+import com.here.ort.utils.log
+
+import javax.script.Compilable
+import javax.script.ScriptEngineManager
+import javax.script.ScriptException
+
+import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
+
+class Evaluator {
+    private val engine = ScriptEngineManager().getEngineByExtension("kts")
+
+    private val preface = """
+            import com.here.ort.model.Error
+            import com.here.ort.model.OrtResult
+
+            val ortResult = bindings["ortResult"] as OrtResult
+            val evalErrors = mutableListOf<Error>()
+
+        """.trimIndent()
+
+    private val postface = """
+
+            evalErrors
+        """.trimIndent()
+
+    init {
+        // This is required to avoid
+        //
+        //     WARN: Failed to initialize native filesystem for Windows
+        //     java.lang.RuntimeException: Could not find installation home path. Please make sure bin/idea.properties
+        //     is present in the installation directory.
+        //
+        // on Windows for some reason.
+        setIdeaIoUseFallback()
+    }
+
+    private fun buildScript(rulesScript: String) = preface + rulesScript + postface
+
+    fun checkSyntax(ortResult: OrtResult, rulesScript: String): Boolean {
+        require(engine is Compilable) {
+            "The scripting engine does not support compilation."
+        }
+
+        engine.put("ortResult", ortResult)
+
+        return try {
+            engine.compile(buildScript(rulesScript))
+            true
+        } catch (e: ScriptException) {
+            log.error { e.message ?: "No error message available." }
+            false
+        }
+    }
+
+    fun evaluate(ortResult: OrtResult, rulesScript: String): EvaluatorRun {
+        engine.put("ortResult", ortResult)
+
+        @Suppress("UNCHECKED_CAST")
+        val errors = engine.eval(buildScript(rulesScript)) as List<Error>
+
+        return EvaluatorRun(errors)
+    }
+}

evaluator/src/test/kotlin/EvaluatorTest.kt

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2017-2018 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package com.here.ort.evaluator
+
+import com.here.ort.model.OrtResult
+import com.here.ort.model.Repository
+import com.here.ort.model.VcsInfo
+import com.here.ort.model.config.RepositoryConfiguration
+
+import io.kotlintest.matchers.haveSize
+import io.kotlintest.should
+import io.kotlintest.shouldBe
+import io.kotlintest.specs.WordSpec
+
+class EvaluatorTest : WordSpec() {
+    init {
+        val ortResult = OrtResult(Repository(VcsInfo.EMPTY, VcsInfo.EMPTY, RepositoryConfiguration()))
+
+        "checkSyntax" should {
+            "succeed if the script can be compiled" {
+                val script = javaClass.getResource("/rules/no_gpl_declared.kts").readText()
+
+                val result = Evaluator().checkSyntax(ortResult, script)
+
+                result shouldBe true
+            }
+
+            "fail if the script can not be compiled" {
+                val result = Evaluator().checkSyntax(ortResult, """
+                    broken script
+                    """.trimIndent())
+
+                result shouldBe false
+            }
+        }
+
+        "evaluate" should {
+            "contain rule errors in the result" {
+                val result = Evaluator().evaluate(ortResult, """
+                    evalErrors += Error(source = "source 1", message = "message 1")
+                    evalErrors += Error(source = "source 2", message = "message 2")
+                    """.trimIndent())
+
+                result.errors should haveSize(2)
+                result.errors[0].let {
+                    it.source shouldBe "source 1"
+                    it.message shouldBe "message 1"
+                }
+                result.errors[1].let {
+                    it.source shouldBe "source 2"
+                    it.message shouldBe "message 2"
+                }
+            }
+        }
+    }
+}

settings.gradle

@@ -25,6 +25,7 @@ rootProject.name = 'oss-review-toolkit'
 include 'analyzer'
 include 'cli'
 include 'downloader'
+include 'evaluator'
 include 'model'
 include 'reporter'
 include 'reporter-web-app'