Skip to content
Permalink
Browse files

evaluator: Switch from `OrtIssue` to `RuleViolation`

Use the `RuleViolation` class as output of the evaluator script. This
provides more fine-grained information about the violations and allows to
improve the reports. Show the data already in the static HTML report,
but not yet in the web app reporter.

Signed-off-by: Martin Nonnenmacher <martin.nonnenmacher@here.com>
  • Loading branch information...
mnonnenmacher committed May 3, 2019
1 parent ce0f3d3 commit 9537d616ea80584d3648b32c4924bbbebef08ca7
@@ -27,9 +27,9 @@ import com.beust.jcommander.Parameters

import com.here.ort.CommandWithHelp
import com.here.ort.evaluator.Evaluator
import com.here.ort.model.OrtIssue
import com.here.ort.model.OrtResult
import com.here.ort.model.OutputFormat
import com.here.ort.model.RuleViolation
import com.here.ort.model.Severity
import com.here.ort.model.mapper
import com.here.ort.model.readValue
@@ -132,12 +132,12 @@ object EvaluatorCommand : CommandWithHelp() {
val evaluatorRun by lazy { evaluator.run(script) }

if (log.isErrorEnabled) {
evaluatorRun.errors.forEach { error ->
log.error(error.toString())
evaluatorRun.violations.forEach { violation ->
log.error(violation.toString())
}
}

printSummary(evaluatorRun.errors)
printSummary(evaluatorRun.violations)

if (absoluteOutputDir != null) {
// Note: This overwrites any existing EvaluatorRun from the input file.
@@ -153,10 +153,10 @@ object EvaluatorCommand : CommandWithHelp() {
}
}

return if (evaluatorRun.errors.isEmpty()) 0 else 2
return if (evaluatorRun.violations.isEmpty()) 0 else 2
}

private fun printSummary(errors: List<OrtIssue>) {
private fun printSummary(errors: List<RuleViolation>) {
val counts = errors.groupingBy { it.severity }.eachCount()

val errorCount = counts[Severity.ERROR] ?: 0
@@ -20,8 +20,8 @@
package com.here.ort.evaluator

import com.here.ort.model.EvaluatorRun
import com.here.ort.model.OrtIssue
import com.here.ort.model.OrtResult
import com.here.ort.model.RuleViolation
import com.here.ort.utils.ScriptRunner

import java.time.Instant
@@ -40,13 +40,13 @@ class Evaluator(ortResult: OrtResult) : ScriptRunner() {
val ortResult = bindings["ortResult"] as OrtResult
// Output:
val evalErrors = mutableListOf<OrtIssue>()
val ruleViolations = mutableListOf<RuleViolation>()
""".trimIndent()

override val postface = """
evalErrors
ruleViolations
""".trimIndent()

init {
@@ -57,10 +57,10 @@ class Evaluator(ortResult: OrtResult) : ScriptRunner() {
val startTime = Instant.now()

@Suppress("UNCHECKED_CAST")
val errors = super.run(script) as List<OrtIssue>
val violations = super.run(script) as List<RuleViolation>

val endTime = Instant.now()

return EvaluatorRun(startTime, endTime, errors)
return EvaluatorRun(startTime, endTime, violations)
}
}
@@ -24,6 +24,7 @@ import com.here.ort.model.LicenseFinding
import com.here.ort.model.LicenseSource
import com.here.ort.model.Package
import com.here.ort.model.Project
import com.here.ort.model.Severity
import com.here.ort.model.config.Excludes
import com.here.ort.model.config.PathExclude
import com.here.ort.spdx.SpdxLicense
@@ -126,6 +127,23 @@ open class PackageRule(
}
}

fun issue(severity: Severity, message: String) = issue(severity, pkg.id, null, null, message)

/**
* Add a [hint][Severity.HINT] to the list of [issues].
*/
fun hint(message: String) = hint(pkg.id, null, null, message)

/**
* Add a [warning][Severity.WARNING] to the list of [issues].
*/
fun warning(message: String) = warning(pkg.id, null, null, message)

/**
* Add an [error][Severity.ERROR] to the list of [issues].
*/
fun error(message: String) = error(pkg.id, null, null, message)

/**
* A [Rule] to check a single license of the [package][pkg].
*/
@@ -186,5 +204,22 @@ open class PackageRule(

override fun matches() = SpdxLicense.forId(license) != null
}

fun issue(severity: Severity, message: String) = issue(severity, pkg.id, license, licenseSource, message)

/**
* Add a [hint][Severity.HINT] to the list of [issues].
*/
fun hint(message: String) = hint(pkg.id, license, licenseSource, message)

/**
* Add a [warning][Severity.WARNING] to the list of [issues].
*/
fun warning(message: String) = warning(pkg.id, license, licenseSource, message)

/**
* Add an [error][Severity.ERROR] to the list of [issues].
*/
fun error(message: String) = error(pkg.id, license, licenseSource, message)
}
}
@@ -21,7 +21,10 @@ package com.here.ort.evaluator

import ch.frankel.slf4k.*

import com.here.ort.model.Identifier
import com.here.ort.model.LicenseSource
import com.here.ort.model.OrtIssue
import com.here.ort.model.RuleViolation
import com.here.ort.model.Severity
import com.here.ort.utils.log

@@ -50,7 +53,7 @@ abstract class Rule(
/**
* The list of all issues created by this rule.
*/
val issues = mutableListOf<OrtIssue>()
val violations = mutableListOf<RuleViolation>()

/**
* Return a human-readable description of this rule.
@@ -75,10 +78,12 @@ abstract class Rule(
log.info { description }

if (matches()) {
ruleSet.issues += issues
ruleSet.violations += violations

if (issues.isNotEmpty()) {
log.info { "\tFound issues:\n\t\t${issues.joinToString("\n\t\t") { "${it.severity}: ${it.message}" }}" }
if (violations.isNotEmpty()) {
log.info {
"\tFound violations:\n\t\t${violations.joinToString("\n\t\t") { "${it.severity}: ${it.message}" }}"
}
}

runInternal()
@@ -102,24 +107,34 @@ abstract class Rule(
*/
abstract fun issueSource(): String

fun issue(severity: Severity, message: String) {
issues += OrtIssue(source = issueSource(), severity = severity, message = message)
fun issue(severity: Severity, pkgId: Identifier, license: String?, licenseSource: LicenseSource?, message: String) {
violations += RuleViolation(
severity = severity,
rule = name,
pkg = pkgId,
license = license,
licenseSource = licenseSource,
message = message
)
}

/**
* Add a [hint][Severity.HINT] to the list of [issues].
*/
fun hint(message: String) = issue(Severity.HINT, message)
fun hint(pkgId: Identifier, license: String?, licenseSource: LicenseSource?, message: String) =
issue(Severity.HINT, pkgId, license, licenseSource, message)

/**
* Add a [warning][Severity.WARNING] to the list of [issues].
*/
fun warning(message: String) = issue(Severity.WARNING, message)
fun warning(pkgId: Identifier, license: String?, licenseSource: LicenseSource?, message: String) =
issue(Severity.WARNING, pkgId, license, licenseSource, message)

/**
* Add an [error][Severity.ERROR] to the list of [issues].
*/
fun error(message: String) = issue(Severity.ERROR, message)
fun error(pkgId: Identifier, license: String?, licenseSource: LicenseSource?, message: String) =
issue(Severity.ERROR, pkgId, license, licenseSource, message)

/**
* A DSL helper class, providing convenience functions for adding [RuleMatcher]s to this rule.
@@ -21,11 +21,11 @@ package com.here.ort.evaluator

import com.here.ort.model.Identifier
import com.here.ort.model.LicenseFinding
import com.here.ort.model.OrtIssue
import com.here.ort.model.OrtResult
import com.here.ort.model.Package
import com.here.ort.model.PackageReference
import com.here.ort.model.Project
import com.here.ort.model.RuleViolation
import com.here.ort.model.Scope

/**
@@ -35,7 +35,7 @@ class RuleSet(val ortResult: OrtResult) {
/**
* The list of all issues created by the rules of this [RuleSet].
*/
val issues = mutableSetOf<OrtIssue>()
val violations = mutableSetOf<RuleViolation>()

/**
* The map of all [LicenseFinding]s and associated path excludes by [Identifier].
@@ -2,13 +2,28 @@
//
// $ ort evaluate -i scanner/src/funTest/assets/file-counter-expected-output-for-analyzer-result.yml --rules-resource /rules/no_gpl_declared.kts

val pkgWithGpl = ortResult.analyzer?.result?.packages?.filter { (pkg, _) ->
pkg.declaredLicenses.any { license ->
license.startsWith("GPL")
// Define a custom rule matcher.
fun PackageRule.LicenseRule.isGpl() =
object : RuleMatcher {
override val description = "isGpl($license)"

override fun matches() = license.contains("GPL")
}

// Define the rule set.
val ruleSet = ruleSet(ortResult) {
// Define a rule that is executed for each package.
packageRule("NO_GPL") {
// Define a rule that is executed for each license of the package.
licenseRule("NO_GPL", LicenseView.All) {
require {
+isGpl()
}

error("The package '${pkg.id.toCoordinates()}' has the ${licenseSource.name} license '$license'.")
}
}
}

// Populate the list of errors to return.
pkgWithGpl?.forEach { (pkg, _) ->
evalErrors += OrtIssue(source = pkg.id.toString(), message = "This package is declared under a GPL license.")
}
ruleViolations += ruleSet.violations
@@ -19,7 +19,10 @@

package com.here.ort.evaluator

import com.here.ort.model.Identifier
import com.here.ort.model.LicenseSource
import com.here.ort.model.OrtResult
import com.here.ort.model.Severity

import io.kotlintest.matchers.beEmpty
import io.kotlintest.matchers.haveSize
@@ -53,25 +56,50 @@ class EvaluatorTest : WordSpec() {
"return no errors for an empty script" {
val result = Evaluator(OrtResult.EMPTY).run("")

result.errors should beEmpty()
result.violations should beEmpty()
}

"contain rule errors in the result" {
val result = Evaluator(OrtResult.EMPTY).run(
"""
evalErrors += OrtIssue(source = "source 1", message = "message 1")
evalErrors += OrtIssue(source = "source 2", message = "message 2")
ruleViolations += RuleViolation(
rule = "rule 1",
pkg = Identifier("type:namespace:name:1.0"),
license = "license 1",
licenseSource = LicenseSource.DETECTED,
severity = Severity.ERROR,
message = "message 1"
)
ruleViolations += RuleViolation(
rule = "rule 2",
pkg = Identifier("type:namespace:name:2.0"),
license = "license 2",
licenseSource = LicenseSource.DECLARED,
severity = Severity.WARNING,
message = "message 2"
)
""".trimIndent()
)

result.errors should haveSize(2)
result.errors[0].let {
it.source shouldBe "source 1"
it.message shouldBe "message 1"
result.violations should haveSize(2)

with(result.violations[0]) {
rule shouldBe "rule 1"
pkg shouldBe Identifier("type:namespace:name:1.0")
license shouldBe "license 1"
licenseSource shouldBe LicenseSource.DETECTED
severity shouldBe Severity.ERROR
message shouldBe "message 1"
}
result.errors[1].let {
it.source shouldBe "source 2"
it.message shouldBe "message 2"

with(result.violations[1]) {
rule shouldBe "rule 2"
pkg shouldBe Identifier("type:namespace:name:2.0")
license shouldBe "license 2"
licenseSource shouldBe LicenseSource.DECLARED
severity shouldBe Severity.WARNING
message shouldBe "message 2"
}
}
}
@@ -35,5 +35,8 @@ data class EvaluatorRun(
*/
val endTime: Instant = Instant.EPOCH,

val errors: List<OrtIssue>
/**
* The list of [RuleViolation]s found by the evaluator.
*/
val violations: List<RuleViolation>
)
@@ -21,7 +21,7 @@ package com.here.ort.model.config

import com.fasterxml.jackson.annotation.JsonIgnore

import com.here.ort.model.OrtIssue
import com.here.ort.model.RuleViolation

/**
* Defines the resolution of a rule violation. This can be used to silence rule violations that have been identified
@@ -51,5 +51,5 @@ data class RuleViolationResolution(
/**
* True if [message] matches the message of [error].
*/
fun matches(error: OrtIssue) = regex.matches(error.message)
fun matches(violation: RuleViolation) = regex.matches(violation.message)
}
@@ -31,7 +31,7 @@ class EvaluatorRunTest : StringSpec() {
"EvaluatorRun without timestamps can be deserialized" {
val yaml = """
---
errors: []
violations: []
""".trimIndent()

val evaluatorRun = yamlMapper.readValue<EvaluatorRun>(yaml)
@@ -45,7 +45,7 @@ class EvaluatorRunTest : StringSpec() {
---
start_time: "1970-01-01T00:00:10Z"
end_time: "1970-01-01T00:00:10Z"
errors: []
violations: []
""".trimIndent()

val evaluatorRun = yamlMapper.readValue<EvaluatorRun>(yaml)
@@ -702,8 +702,8 @@ function* convertReportData() {
repository: reportData.repository || {},
violations: {
resolved: addKeyToArrayItems([]),
open: (reportData.evaluator && reportData.evaluator.errors)
? addKeyToArrayItems(reportData.evaluator.errors) : []
open: (reportData.evaluator && reportData.evaluator.violations)
? addKeyToArrayItems(reportData.evaluator.violations) : []
}
};

Oops, something went wrong.

0 comments on commit 9537d61

Please sign in to comment.
You can’t perform that action at this time.