rubysrc2cpg: supports regexp literals, and refactors literals support…

… in general (#2818) * RubyParser: name literal alternatives * Refactor astForLiteralPrimaryContext and supported regexp literals * scalafmt * add some float literal tests * scalafmt
joernio · Jun 6, 2023 · 760168d · 760168d
1 parent ce33885
commit 760168d
Show file tree

Hide file tree

Showing 11 changed files with 224 additions and 87 deletions.
diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4
@@ -516,11 +516,11 @@ scopedConstantReference
 // --------------------------------------------------------
 
 literal
-    :   numericLiteral
-    |   symbol
-    |   SINGLE_QUOTED_STRING_LITERAL
-    |   DOUBLE_QUOTED_STRING_START DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE? DOUBLE_QUOTED_STRING_END
-    |   REGULAR_EXPRESSION_START REGULAR_EXPRESSION_BODY? REGULAR_EXPRESSION_END
+    :   numericLiteral                                                                                              # numericLiteralLiteral
+    |   symbol                                                                                                      # symbolLiteral
+    |   SINGLE_QUOTED_STRING_LITERAL                                                                                # singleQuotedStringLiteral
+    |   DOUBLE_QUOTED_STRING_START DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE? DOUBLE_QUOTED_STRING_END                # doubleQuotedStringLiteral
+    |   REGULAR_EXPRESSION_START REGULAR_EXPRESSION_BODY? REGULAR_EXPRESSION_END                                    # regularExpressionLiteral
     ;
 
 symbol

diff --git a/...li/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/...li/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala
@@ -301,7 +301,7 @@ class AstCreator(filename: String, global: Global)
       astForChainedScopedConstantReferencePrimaryContext(ctx)
     case ctx: ArrayConstructorPrimaryContext          => astForArrayConstructorPrimaryContext(ctx)
     case ctx: HashConstructorPrimaryContext           => astForHashConstructorPrimaryContext(ctx)
-    case ctx: LiteralPrimaryContext                   => astForLiteralPrimaryContext(ctx)
+    case ctx: LiteralPrimaryContext                   => Seq(astForLiteralPrimaryContext(ctx))
     case ctx: StringInterpolationPrimaryContext       => astForStringInterpolationPrimaryContext(ctx)
     case ctx: IsDefinedPrimaryContext                 => astForIsDefinedPrimaryContext(ctx)
     case ctx: SuperExpressionPrimaryContext           => astForSuperExpressionPrimaryContext(ctx)
@@ -1138,44 +1138,14 @@ class AstCreator(filename: String, global: Global)
     }
   }
 
-  def astForLiteralPrimaryContext(ctx: LiteralPrimaryContext): Seq[Ast] = {
-    val lineStart   = line(ctx.literal())
-    val columnStart = column(ctx.literal())
-    if (ctx.literal().numericLiteral() != null) {
-      val text = ctx.getText
-      val node = NewLiteral()
-        .code(text)
-        .lineNumber(lineStart)
-        .columnNumber(columnStart)
-        .typeFullName(Defines.Numeric)
-        .dynamicTypeHintFullName(List(Defines.Numeric))
-      registerType(Defines.Numeric)
-      Seq(Ast(node))
-    } else if (ctx.literal().SINGLE_QUOTED_STRING_LITERAL() != null) {
-      val text = ctx.getText
-      val node = NewLiteral()
-        .code(text)
-        .lineNumber(lineStart)
-        .columnNumber(columnStart)
-        .typeFullName(Defines.String)
-        .dynamicTypeHintFullName(List(Defines.String))
-      Seq(Ast(node))
-    } else if (ctx.literal().DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE() != null) {
-      val text = ctx.literal().DOUBLE_QUOTED_STRING_CHARACTER_SEQUENCE().getText
-      val node = NewLiteral()
-        .code(text)
-        .lineNumber(lineStart)
-        .columnNumber(columnStart)
-        .typeFullName(Defines.String)
-        .dynamicTypeHintFullName(List(Defines.String))
-      registerType(Defines.String)
-      Seq(Ast(node))
-    } else if (ctx.literal().symbol() != null) {
-      astForSymbolContext(ctx.literal().symbol())
-    } else {
-      Seq(Ast())
+  def astForLiteralPrimaryContext(ctx: LiteralPrimaryContext): Ast =
+    ctx.literal() match {
+      case ctx: NumericLiteralLiteralContext     => astForNumericLiteral(ctx.numericLiteral)
+      case ctx: SymbolLiteralContext             => astForSymbolLiteral(ctx.symbol())
+      case ctx: SingleQuotedStringLiteralContext => astForSingleQuotedStringLiteral(ctx)
+      case ctx: DoubleQuotedStringLiteralContext => astForDoubleQuotedStringLiteral(ctx)
+      case ctx: RegularExpressionLiteralContext  => astForRegularExpressionLiteral(ctx)
     }
-  }
 
   def astForSimpleMethodNamePartContext(ctx: SimpleMethodNamePartContext): Seq[Ast] = {
     astForDefinedMethodNameContext(ctx.definedMethodName())

diff --git a/...rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForPrimitivesCreator.scala b/...rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForPrimitivesCreator.scala
@@ -26,4 +26,25 @@ trait AstForPrimitivesCreator { this: AstCreator =>
 
   protected def astForEncodingPseudoIdentifier(ctx: RubyParser.EncodingPseudoVariableIdentifierContext): Ast =
     Ast(createIdentifierWithScope(ctx, ctx.getText, ctx.getText, Defines.Encoding))
+
+  protected def astForNumericLiteral(ctx: RubyParser.NumericLiteralContext): Ast = {
+    val numericTypeName = if (isFloatLiteral(ctx.unsignedNumericLiteral)) Defines.Float else Defines.Integer
+    Ast(literalNode(ctx, ctx.getText, numericTypeName))
+  }
+
+  protected def astForSymbolLiteral(ctx: RubyParser.SymbolContext): Ast =
+    Ast(literalNode(ctx, ctx.getText, Defines.Symbol))
+
+  protected def astForSingleQuotedStringLiteral(ctx: RubyParser.SingleQuotedStringLiteralContext): Ast =
+    Ast(literalNode(ctx, ctx.getText, Defines.String))
+
+  protected def astForDoubleQuotedStringLiteral(ctx: RubyParser.DoubleQuotedStringLiteralContext): Ast =
+    Ast(literalNode(ctx, ctx.getText, Defines.String))
+
+  protected def astForRegularExpressionLiteral(ctx: RubyParser.RegularExpressionLiteralContext): Ast =
+    Ast(literalNode(ctx, ctx.getText, Defines.Regexp))
+
+  private def isFloatLiteral(ctx: RubyParser.UnsignedNumericLiteralContext): Boolean =
+    Option(ctx.FLOAT_LITERAL_WITH_EXPONENT).isDefined || Option(ctx.FLOAT_LITERAL_WITHOUT_EXPONENT).isDefined
+
 }
diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/Defines.scala
@@ -19,6 +19,7 @@ object Defines {
   val Hash: String  = "Hash"
 
   val Encoding: String = "Encoding"
+  val Regexp: String   = "Regexp"
 
   // TODO: The following shall be moved out eventually.
   val ModifierRedo: String  = "redo"

diff --git a/...bysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesesTests.scala b/...bysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesesTests.scala
@@ -47,7 +47,7 @@ class InvocationWithParenthesesTests extends RubyParserAbstractTest {
             |   Expressions
             |    PrimaryExpression
             |     LiteralPrimary
-            |      Literal
+            |      NumericLiteralLiteral
             |       NumericLiteral
             |        UnsignedNumericLiteral
             |         1
@@ -75,7 +75,7 @@ class InvocationWithParenthesesTests extends RubyParserAbstractTest {
             |     WsOrNl
             |     PrimaryExpression
             |      LiteralPrimary
-            |       Literal
+            |       NumericLiteralLiteral
             |        NumericLiteral
             |         UnsignedNumericLiteral
             |          1
@@ -95,7 +95,7 @@ class InvocationWithParenthesesTests extends RubyParserAbstractTest {
             |   Expressions
             |    PrimaryExpression
             |     LiteralPrimary
-            |      Literal
+            |      SymbolLiteral
             |       Symbol
             |        :region
             |  )""".stripMargin
@@ -114,7 +114,7 @@ class InvocationWithParenthesesTests extends RubyParserAbstractTest {
             |   Expressions
             |    PrimaryExpression
             |     LiteralPrimary
-            |      Literal
+            |      SymbolLiteral
             |       Symbol
             |        :region
             |  ,

diff --git a/...ntends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionTests.scala b/...ntends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionTests.scala
@@ -74,7 +74,7 @@ class MethodDefinitionTests extends RubyParserAbstractTest {
              |      =
              |      PrimaryExpression
              |       LiteralPrimary
-             |        Literal
+             |        NumericLiteralLiteral
              |         NumericLiteral
              |          UnsignedNumericLiteral
              |           1

diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexTests.scala
@@ -10,7 +10,7 @@ class RegexTests extends RubyParserAbstractTest {
       "be parsed as a primary expression" in {
         printAst(_.primary(), code) shouldEqual
           """LiteralPrimary
-            | Literal
+            | RegularExpressionLiteral
             |  /
             |  /""".stripMargin
       }
@@ -33,7 +33,7 @@ class RegexTests extends RubyParserAbstractTest {
             |    ExpressionExpressionOrCommand
             |     PrimaryExpression
             |      LiteralPrimary
-            |       Literal
+            |       RegularExpressionLiteral
             |        /
             |        /""".stripMargin
       }
@@ -78,7 +78,7 @@ class RegexTests extends RubyParserAbstractTest {
             |     Expressions
             |      PrimaryExpression
             |       LiteralPrimary
-            |        Literal
+            |        RegularExpressionLiteral
             |         /
             |         /
             |    )""".stripMargin
@@ -94,7 +94,7 @@ class RegexTests extends RubyParserAbstractTest {
       "be parsed as a primary expression" in {
         printAst(_.primary(), code) shouldEqual
           """LiteralPrimary
-            | Literal
+            | RegularExpressionLiteral
             |  /
             |  (eu|us)
             |  /""".stripMargin
@@ -118,7 +118,7 @@ class RegexTests extends RubyParserAbstractTest {
             |    ExpressionExpressionOrCommand
             |     PrimaryExpression
             |      LiteralPrimary
-            |       Literal
+            |       RegularExpressionLiteral
             |        /
             |        (eu|us)
             |        /""".stripMargin
@@ -165,7 +165,7 @@ class RegexTests extends RubyParserAbstractTest {
             |     Expressions
             |      PrimaryExpression
             |       LiteralPrimary
-            |        Literal
+            |        RegularExpressionLiteral
             |         /
             |         (eu|us)
             |         /
@@ -193,7 +193,7 @@ class RegexTests extends RubyParserAbstractTest {
               |      ExpressionExpressionOrCommand
               |       PrimaryExpression
               |        LiteralPrimary
-              |         Literal
+              |         NumericLiteralLiteral
               |          NumericLiteral
               |           UnsignedNumericLiteral
               |            1
@@ -231,7 +231,7 @@ class RegexTests extends RubyParserAbstractTest {
             |            ExpressionExpressionOrCommand
             |             PrimaryExpression
             |              LiteralPrimary
-            |               Literal
+            |               NumericLiteralLiteral
             |                NumericLiteral
             |                 UnsignedNumericLiteral
             |                  1
@@ -306,7 +306,7 @@ class RegexTests extends RubyParserAbstractTest {
             |             ExpressionExpressionOrCommand
             |              PrimaryExpression
             |               LiteralPrimary
-            |                Literal
+            |                NumericLiteralLiteral
             |                 NumericLiteral
             |                  UnsignedNumericLiteral
             |                   1

diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringTests.scala
@@ -10,7 +10,7 @@ class StringTests extends RubyParserAbstractTest {
       "be parsed as a primary expression" in {
         printAst(_.primary(), code) shouldEqual
           """LiteralPrimary
-            | Literal
+            | SingleQuotedStringLiteral
             |  ''""".stripMargin
       }
     }
@@ -24,7 +24,7 @@ class StringTests extends RubyParserAbstractTest {
       "be parsed as a primary expression" in {
         printAst(_.primary(), code) shouldEqual
           """LiteralPrimary
-            | Literal
+            | DoubleQuotedStringLiteral
             |  "
             |  """".stripMargin
       }
@@ -47,7 +47,7 @@ class StringTests extends RubyParserAbstractTest {
             |      ExpressionExpressionOrCommand
             |       PrimaryExpression
             |        LiteralPrimary
-            |         Literal
+            |         NumericLiteralLiteral
             |          NumericLiteral
             |           UnsignedNumericLiteral
             |            1
@@ -72,7 +72,7 @@ class StringTests extends RubyParserAbstractTest {
             |      ExpressionExpressionOrCommand
             |       PrimaryExpression
             |        LiteralPrimary
-            |         Literal
+            |         NumericLiteralLiteral
             |          NumericLiteral
             |           UnsignedNumericLiteral
             |            1
@@ -85,7 +85,7 @@ class StringTests extends RubyParserAbstractTest {
             |      ExpressionExpressionOrCommand
             |       PrimaryExpression
             |        LiteralPrimary
-            |         Literal
+            |         NumericLiteralLiteral
             |          NumericLiteral
             |           UnsignedNumericLiteral
             |            2

diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/SymbolTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/SymbolTests.scala
@@ -8,7 +8,7 @@ class SymbolTests extends RubyParserAbstractTest {
 
       def symbolLiteralParseTreeText(symbolName: String): String =
         s"""LiteralPrimary
-           | Literal
+           | SymbolLiteral
            |  Symbol
            |   $symbolName""".stripMargin