From b2b5103db25b5b1a2ec53699da892e649528d663 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Wed, 5 Jun 2024 15:42:40 +0200 Subject: [PATCH 1/2] [ruby] Type/Method Identifier Ref Self Access This change moves the type/method identifier references for entities exportable from the script to prefix the respective entity at the definition. --- .../astcreation/AstCreatorHelper.scala | 13 +++++- .../astcreation/AstForFunctionsCreator.scala | 46 +++++++++++++------ .../astcreation/AstForTypesCreator.scala | 41 ++++++++++++----- .../rubysrc2cpg/querying/MethodTests.scala | 41 +++++++++-------- 4 files changed, 95 insertions(+), 46 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala index f77760755dc..f3dd1b7924d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala @@ -87,7 +87,16 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As lineNumber: Option[Integer], columnNumber: Option[Integer] ): Ast = { - val code = Seq(lhs, rhs).collect { case x: AstNodeNew => x.code }.mkString(" = ") + astForAssignment(Ast(lhs), Ast(rhs), lineNumber, columnNumber) + } + + protected def astForAssignment( + lhs: Ast, + rhs: Ast, + lineNumber: Option[Integer], + columnNumber: Option[Integer] + ): Ast = { + val code = Seq(lhs, rhs).flatMap(_.root).collect { case x: ExpressionNew => x.code }.mkString(" = ") val assignment = NewCall() .name(Operators.assignment) .methodFullName(Operators.assignment) @@ -96,7 +105,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As .lineNumber(lineNumber) .columnNumber(columnNumber) - callAst(assignment, Seq(Ast(lhs), Ast(rhs))) + callAst(assignment, Seq(lhs, rhs)) } protected val UnaryOperatorNames: Map[String, String] = Map( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala index ff7173243c1..ade4e54284d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForFunctionsCreator.scala @@ -12,7 +12,14 @@ import io.joern.x2cpg.utils.NodeBuilders.{ } import io.joern.x2cpg.{Ast, AstEdge, ValidationMode, Defines as XDefines} import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, ModifierTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{ + DispatchTypes, + EdgeTypes, + EvaluationStrategies, + ModifierTypes, + NodeTypes, + Operators +} import io.joern.rubysrc2cpg.utils.FreshNameGenerator import scala.collection.mutable @@ -378,19 +385,30 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode) { th private def createMethodRefPointer(method: NewMethod): Ast = { if (scope.isSurroundedByProgramScope) { - val methodRefNode = NewMethodRef() - .code(s"def ${method.name} (...)") - .methodFullName(method.fullName) - .typeFullName(Defines.Any) - .lineNumber(method.lineNumber) - .columnNumber(method.columnNumber) - - val methodRefIdent = NewIdentifier() - .code(method.name) - .name(method.name) - .typeFullName(method.fullName) - .lineNumber(method.lineNumber) - .columnNumber(method.columnNumber) + val methodRefNode = Ast( + NewMethodRef() + .code(s"def ${method.name} (...)") + .methodFullName(method.fullName) + .typeFullName(Defines.Any) + .lineNumber(method.lineNumber) + .columnNumber(method.columnNumber) + ) + + val methodRefIdent = { + val self = NewIdentifier().name(Defines.Self).code(Defines.Self).typeFullName(Defines.Any) + val fi = NewFieldIdentifier() + .code(method.name) + .canonicalName(method.name) + .lineNumber(method.lineNumber) + .columnNumber(method.columnNumber) + val fieldAccess = NewCall() + .name(Operators.fieldAccess) + .code(s"${Defines.Self}.${method.name}") + .methodFullName(Operators.fieldAccess) + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .typeFullName(Defines.Any) + callAst(fieldAccess, Seq(Ast(self), Ast(fi))) + } astForAssignment(methodRefIdent, methodRefNode, method.lineNumber, method.columnNumber) } else { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala index 438ab9cca67..660645978ff 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala @@ -4,7 +4,13 @@ import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* import io.joern.rubysrc2cpg.datastructures.{BlockScope, MethodScope, ModuleScope, TypeScope} import io.joern.rubysrc2cpg.passes.Defines import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} -import io.shiftleft.codepropertygraph.generated.nodes.{NewIdentifier, NewTypeDecl, NewTypeRef} +import io.shiftleft.codepropertygraph.generated.nodes.{ + NewCall, + NewFieldIdentifier, + NewIdentifier, + NewTypeDecl, + NewTypeRef +} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EvaluationStrategies, Operators} import scala.collection.immutable.List @@ -96,18 +102,29 @@ trait AstForTypesCreator(implicit withSchemaValidation: ValidationMode) { this: private def createTypeRefPointer(typeDecl: NewTypeDecl): Ast = { if (scope.isSurroundedByProgramScope) { - val typeRefName = typeDecl.name.split("[.]").takeRight(1).head - val typeRefNode = NewTypeRef() - .code(s"class ${typeDecl.fullName} (...)") - .typeFullName(typeDecl.fullName) - - val typeRefIdent = NewIdentifier() - .code(typeRefName) - .name(typeRefName) - .typeFullName(typeDecl.fullName) - .lineNumber(typeDecl.lineNumber) - .columnNumber(typeDecl.columnNumber) + val typeRefNode = Ast( + NewTypeRef() + .code(s"class ${typeDecl.name} (...)") + .typeFullName(typeDecl.fullName) + .lineNumber(typeDecl.lineNumber) + .columnNumber(typeDecl.columnNumber) + ) + val typeRefIdent = { + val self = NewIdentifier().name(Defines.Self).code(Defines.Self).typeFullName(Defines.Any) + val fi = NewFieldIdentifier() + .code(typeDecl.name) + .canonicalName(typeDecl.name) + .lineNumber(typeDecl.lineNumber) + .columnNumber(typeDecl.columnNumber) + val fieldAccess = NewCall() + .name(Operators.fieldAccess) + .code(s"${Defines.Self}.${typeDecl.name}") + .methodFullName(Operators.fieldAccess) + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .typeFullName(Defines.Any) + callAst(fieldAccess, Seq(Ast(self), Ast(fi))) + } astForAssignment(typeRefIdent, typeRefNode, typeDecl.lineNumber, typeDecl.columnNumber) } else { Ast() diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala index d704f2e165e..81d9aa0f181 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala @@ -592,27 +592,30 @@ class MethodTests extends RubyCode2CpgFixture { "be directly under :program" in { inside(cpg.method.name(RDefines.Program).filename("t1.rb").assignment.l) { case moduleAssignment :: classAssignment :: methodAssignment :: Nil => - moduleAssignment.code shouldBe "A = class t1.rb:::program.A (...)" - classAssignment.code shouldBe "B = class t1.rb:::program.B (...)" - methodAssignment.code shouldBe "c = def c (...)" + moduleAssignment.code shouldBe "self.A = class A (...)" + classAssignment.code shouldBe "self.B = class B (...)" + methodAssignment.code shouldBe "self.c = def c (...)" inside(moduleAssignment.argument.l) { - case (lhs: Identifier) :: (rhs: TypeRef) :: Nil => - lhs.name shouldBe "A" + case (lhs: Call) :: (rhs: TypeRef) :: Nil => + lhs.code shouldBe "self.A" + lhs.name shouldBe Operators.fieldAccess rhs.typeFullName shouldBe "t1.rb:::program.A" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } inside(classAssignment.argument.l) { - case (lhs: Identifier) :: (rhs: TypeRef) :: Nil => - lhs.name shouldBe "B" + case (lhs: Call) :: (rhs: TypeRef) :: Nil => + lhs.code shouldBe "self.B" + lhs.name shouldBe Operators.fieldAccess rhs.typeFullName shouldBe "t1.rb:::program.B" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } inside(methodAssignment.argument.l) { - case (lhs: Identifier) :: (rhs: MethodRef) :: Nil => - lhs.name shouldBe "c" + case (lhs: Call) :: (rhs: MethodRef) :: Nil => + lhs.code shouldBe "self.c" + lhs.name shouldBe Operators.fieldAccess rhs.methodFullName shouldBe "t1.rb:::program:c" rhs.typeFullName shouldBe RDefines.Any case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") @@ -625,19 +628,21 @@ class MethodTests extends RubyCode2CpgFixture { "not be present in other files" in { inside(cpg.method.name(RDefines.Program).filename("t2.rb").assignment.l) { case classAssignment :: methodAssignment :: Nil => - classAssignment.code shouldBe "D = class t2.rb:::program.D (...)" - methodAssignment.code shouldBe "e = def e (...)" + classAssignment.code shouldBe "self.D = class D (...)" + methodAssignment.code shouldBe "self.e = def e (...)" inside(classAssignment.argument.l) { - case (lhs: Identifier) :: (rhs: TypeRef) :: Nil => - lhs.name shouldBe "D" + case (lhs: Call) :: (rhs: TypeRef) :: Nil => + lhs.code shouldBe "self.D" + lhs.name shouldBe Operators.fieldAccess rhs.typeFullName shouldBe "t2.rb:::program.D" case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") } inside(methodAssignment.argument.l) { - case (lhs: Identifier) :: (rhs: MethodRef) :: Nil => - lhs.name shouldBe "e" + case (lhs: Call) :: (rhs: MethodRef) :: Nil => + lhs.code shouldBe "self.e" + lhs.name shouldBe Operators.fieldAccess rhs.methodFullName shouldBe "t2.rb:::program:e" rhs.typeFullName shouldBe RDefines.Any case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") @@ -650,9 +655,9 @@ class MethodTests extends RubyCode2CpgFixture { "be placed directly before each entity's definition" in { inside(cpg.method.name(RDefines.Program).filename("t1.rb").block.astChildren.l) { case (a1: Call) :: (_: TypeDecl) :: (a2: Call) :: (_: TypeDecl) :: (a3: Call) :: (_: Method) :: (_: TypeDecl) :: Nil => - a1.code shouldBe "A = class t1.rb:::program.A (...)" - a2.code shouldBe "B = class t1.rb:::program.B (...)" - a3.code shouldBe "c = def c (...)" + a1.code shouldBe "self.A = class A (...)" + a2.code shouldBe "self.B = class B (...)" + a3.code shouldBe "self.c = def c (...)" case xs => fail(s"Expected assignments to appear before definitions, instead got [$xs]") } } From 333e4921791304168726cc18857ed349ee15ace0 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Wed, 5 Jun 2024 16:10:02 +0200 Subject: [PATCH 2/2] Adjusted test expectations --- .../dataflow/MethodReturnTests.scala | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/MethodReturnTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/MethodReturnTests.scala index 2e0afc95f8d..3649d9f47f7 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/MethodReturnTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/MethodReturnTests.scala @@ -81,8 +81,12 @@ class MethodReturnTests extends RubyCode2CpgFixture(withPostProcessing = true, w ("arg > 1", 3), ("arg + 1", 4), ("RET", 2), + ("self.foo = def foo (...)", 2), + ("self.foo = def foo (...)", -1), + ("foo x", 11), + ("foo(self, arg)", 2), + ("RET", 2), ("foo x", 11), - ("y = foo x", 11), ("puts y", 12) ) ) @@ -122,8 +126,12 @@ class MethodReturnTests extends RubyCode2CpgFixture(withPostProcessing = true, w ("RET", 2), ("add(arg)", 8), ("RET", 6), + ("self.foo = def foo (...)", 6), + ("self.foo = def foo (...)", -1), + ("foo x", 15), + ("foo(self, arg)", 6), + ("RET", 6), ("foo x", 15), - ("y = foo x", 15), ("puts y", 16) ) ) @@ -151,8 +159,12 @@ class MethodReturnTests extends RubyCode2CpgFixture(withPostProcessing = true, w ("q = p", 3), ("return q", 4), ("RET", 2), + ("self.add = def add (...)", 2), + ("self.add = def add (...)", -1), + ("add(n)", 8), + ("add(self, p)", 2), + ("RET", 2), ("add(n)", 8), - ("ret = add(n)", 8), ("puts ret", 9) ) )