Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

minor: Fix sonarcube warnings #736

Merged
merged 2 commits into from
Aug 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .sonarcloud.properties
Original file line number Diff line number Diff line change
@@ -1 +1 @@
sonar.cpd.exclusions=/core/src/test/**/*.scala,integration-tests/src/test/**/*.scala
sonar.cpd.exclusions=**/*Spec.*,**/*Test.*,**/*Tests.*
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,8 @@ object HadoopConfigurationTest {
Configuration.addDefaultResource(
s"${this.getClass.getPackage.getName.replaceAllLiterally(".", "/")}/hdfs-test-conf.xml")

def init(): Unit = {}
def init(): Unit = {
// A singleton activator.
// Called from the class instances to ensure execution of the companion object's constructor code.
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,12 @@ class DataSourcePasswordReplacingFilterSpec extends AnyFlatSpec with Matchers wi
"b" -> Seq(null),
"c" -> None,
"m" -> Map(
"url" -> "jdbc:postgresql://bob:secret@someHost:somePort/someDB",
"url" -> "jdbc:postgresql://bob:secret@someHost:somePort/someDB", // NOSONAR
"dbtable" -> "someTable",
"user" -> "someUser",
"PASSPHRASE" -> "somePassword" // NOSONAR
),
"url" -> "jdbc:postgresql://bob:secret@someHost:somePort/someDB",
"url" -> "jdbc:postgresql://bob:secret@someHost:somePort/someDB", // NOSONAR
"dbtable" -> "someTable",
"user" -> "someUser",
"password" -> "somePassword" // NOSONAR
Expand All @@ -92,14 +92,14 @@ class DataSourcePasswordReplacingFilterSpec extends AnyFlatSpec with Matchers wi
"a" -> 42,
"b" -> Seq(null),
"c" -> None,
"url" -> "jdbc:postgresql://bob:*****@someHost:somePort/someDB",
"url" -> "jdbc:postgresql://bob:*****@someHost:somePort/someDB", // NOSONAR
"m" -> Map(
"url" -> "jdbc:postgresql://bob:*****@someHost:somePort/someDB",
"url" -> "jdbc:postgresql://bob:*****@someHost:somePort/someDB", // NOSONAR
"dbtable" -> "someTable",
"user" -> "someUser",
"PASSPHRASE" -> "*****" // NOSONAR
),
"url" -> "jdbc:postgresql://bob:*****@someHost:somePort/someDB",
"url" -> "jdbc:postgresql://bob:*****@someHost:somePort/someDB", // NOSONAR
"dbtable" -> "someTable",
"user" -> "someUser",
"password" -> "*****" // NOSONAR
Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/scala/za/co/absa/spline/SparkApp.scala
Original file line number Diff line number Diff line change
Expand Up @@ -47,5 +47,5 @@ abstract class SparkApp
*/
val spark: SparkSession = sparkBuilder.getOrCreate()

protected override def _sqlContext: SQLContext = spark.sqlContext
protected override def _sqlContext: SQLContext = spark.sqlContext // NOSONAR
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,40 +18,68 @@ package za.co.absa.spline.test

import za.co.absa.spline.producer.model._

/**
* A class to walk through the execution plan.
*
* @param allOpMap A map of all operations in the execution plan.
* @param funMap A map of all functional expressions in the execution plan.
* @param litMap A map of all literals in the execution plan.
* @param attrMap A map of all attributes in the execution plan.
*/
class LineageWalker(
allOpMap: Map[String, Operation],
funMap: Map[String, FunctionalExpression],
litMap: Map[String, Literal],
attrMap: Map[String, Attribute]
) {

def attributeById(attributeId: String): Attribute = attrMap(attributeId)

def operationById(operationId: String): Operation = allOpMap(operationId)

def dependsOn(att: Attribute, onAtt: Attribute): Boolean = {
dependsOnRec(AttrRef(att.id), onAtt.id)
/**
* Retrieves an attribute by its ID.
*
* @param attributeId The ID of the attribute.
* @return The attribute with the given ID.
*/
def getAttributeById(attributeId: String): Attribute = attrMap(attributeId)

/**
* Retrieves an operation by its ID.
*
* @param operationId The ID of the operation.
* @return The operation with the given ID.
*/
def getOperationById(operationId: String): Operation = allOpMap(operationId)

/**
* Checks if an attribute depends on another attribute.
*
* @param sourceAttribute The attribute to check.
* @param targetAttribute The attribute that the first attribute may depend on.
* @return True if the first attribute depends on the second attribute, false otherwise.
*/
def dependsOn(sourceAttribute: Attribute, targetAttribute: Attribute): Boolean = {
dependsOnRecursively(AttrRef(sourceAttribute.id), targetAttribute.id)
}

private def dependsOnRec(refs: Seq[AttrOrExprRef], id: String): Boolean =
refs.exists(dependsOnRec(_, id))
private def dependsOnRecursively(refs: Seq[AttrOrExprRef], attrId: String): Boolean =
refs.exists(dependsOnRecursively(_, attrId))

private def dependsOnRec(ref: AttrOrExprRef, id: String): Boolean = ref match {
case AttrRef(attrIfd) =>
if (attrIfd == id) true
else dependsOnRec(attrMap(attrIfd).childRefs, id)
private def dependsOnRecursively(ref: AttrOrExprRef, targetAttrId: String): Boolean = ref match {
case AttrRef(attrId) =>
attrId == targetAttrId || dependsOnRecursively(attrMap(attrId).childRefs, targetAttrId)
case ExprRef(exprId) =>
if (exprId == id) true
else {
if (litMap.contains("exprId")) false
else dependsOnRec(funMap(exprId).childRefs, id)
}
funMap.get(exprId).exists(expr => dependsOnRecursively(expr.childRefs, targetAttrId))
}

}

object LineageWalker {

/**
* Creates a LineageWalker instance from an execution plan.
*
* @param plan The execution plan.
* @return A LineageWalker instance.
*/
def apply(plan: ExecutionPlan): LineageWalker = {
val allOpMap = plan.operations.all
.map(op => op.id -> op)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ object ProducerModelImplicits {
implicit class OperationOps(val operation: Operation) extends AnyVal {

def outputAttributes(implicit walker: LineageWalker): IOAttributes = {
operation.output.map(walker.attributeById)
operation.output.map(walker.getAttributeById)
}

def childOperations(implicit walker: LineageWalker): Seq[Operation] = {
operation.childIds.map(walker.operationById)
operation.childIds.map(walker.getOperationById)
}

def childOperation(implicit walker: LineageWalker): Operation = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

package za.co.absa.spline.test.fixture.spline

import org.apache.commons.configuration.BaseConfiguration
import org.apache.spark.sql.SparkSession


Expand All @@ -26,8 +25,6 @@ trait SplineFixture {
}

object SplineFixture {
def EmptyConf = new BaseConfiguration

def extractTableIdentifier(params: Map[String, Any]): Map[String, Any] =
params
.apply("table").asInstanceOf[Map[String, _]]
Expand Down
2 changes: 0 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,6 @@
<arg>-deprecation</arg>
<arg>-unchecked</arg>
<arg>-Ywarn-numeric-widen</arg>
<!--<arg>-Ywarn-dead-code</arg>-->
<!--<arg>-Ywarn-value-discard</arg>-->
</args>
</configuration>
<executions>
Expand Down
Loading