Make the filename property in pysrc2cpg relative again. (#3544)

This the introduction of META_DATA.root, absolute path are not longer required.
joernio · Aug 24, 2023 · 86c3763 · 86c3763
1 parent b8f9812
commit 86c3763
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 24 deletions.
diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/CodeToCpg.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/CodeToCpg.scala
@@ -20,15 +20,13 @@ class CodeToCpg(cpg: Cpg, inputProvider: Iterable[InputProvider], schemaValidati
       val lineBreakCorrectedCode = inputPair.content.replace("\r\n", "\n").replace("\r", "\n")
       val astRoot                = parser.parse(lineBreakCorrectedCode)
       val nodeToCode             = new NodeToCode(lineBreakCorrectedCode)
-      val astVisitor = new PythonAstVisitor(inputPair.absFileName, inputPair.relFileName, nodeToCode, PythonV2AndV3)(
-        schemaValidationMode
-      )
+      val astVisitor = new PythonAstVisitor(inputPair.relFileName, nodeToCode, PythonV2AndV3)(schemaValidationMode)
       astVisitor.convert(astRoot)
 
       diffGraph.absorb(astVisitor.getDiffGraph)
     } catch {
       case exception: Throwable =>
-        logger.warn(s"Failed to convert file ${inputPair.absFileName}", exception)
+        logger.warn(s"Failed to convert file ${inputPair.relFileName}", exception)
         Iterator.empty
     }
   }

diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2Cpg.scala
@@ -7,7 +7,7 @@ import overflowdb.BatchedUpdate
 import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 object Py2Cpg {
-  case class InputPair(content: String, absFileName: String, relFileName: String)
+  case class InputPair(content: String, relFileName: String)
   type InputProvider = () => InputPair
 }
 

diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/Py2CpgOnFileSystem.scala
@@ -55,7 +55,7 @@ class Py2CpgOnFileSystem extends X2CpgFrontend[Py2CpgOnFileSystemConfig] {
       val inputProviders = inputFiles.map { inputFile => () =>
         {
           val content = IOUtils.readLinesInFile(inputFile).mkString("\n")
-          Py2Cpg.InputPair(content, inputFile.toString, Paths.get(config.inputPath).relativize(inputFile).toString)
+          Py2Cpg.InputPair(content, Paths.get(config.inputPath).relativize(inputFile).toString)
         }
       }
       val py2Cpg = new Py2Cpg(inputProviders, cpg, config.inputPath, config.requirementsTxt, config.schemaValidation)

diff --git a/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonAstVisitor.scala
@@ -22,13 +22,9 @@ object PythonV2      extends PythonVersion
 object PythonV3      extends PythonVersion
 object PythonV2AndV3 extends PythonVersion
 
-class PythonAstVisitor(
-  absFileName: String,
-  relFileName: String,
-  protected val nodeToCode: NodeToCode,
-  version: PythonVersion
-)(implicit withSchemaValidation: ValidationMode)
-    extends PythonAstVisitorHelpers {
+class PythonAstVisitor(relFileName: String, protected val nodeToCode: NodeToCode, version: PythonVersion)(implicit
+  withSchemaValidation: ValidationMode
+) extends PythonAstVisitorHelpers {
 
   private val diffGraph     = new DiffGraphBuilder()
   protected val nodeBuilder = new NodeBuilder(diffGraph)
@@ -76,12 +72,12 @@ class PythonAstVisitor(
     module.accept(memOpCalculator)
     memOpMap = memOpCalculator.astNodeToMemOp
 
-    val fileNode = nodeBuilder.fileNode(absFileName)
+    val fileNode = nodeBuilder.fileNode(relFileName)
     val namespaceBlockNode =
       nodeBuilder.namespaceBlockNode(
         Constants.GLOBAL_NAMESPACE,
         relFileName + ":" + Constants.GLOBAL_NAMESPACE,
-        absFileName
+        relFileName
       )
     edgeBuilder.astEdge(namespaceBlockNode, fileNode, 1)
     contextStack.setFileNamespaceBlock(namespaceBlockNode)
@@ -346,7 +342,7 @@ class PythonAstVisitor(
     returnTypeHint: Option[String],
     lineAndColumn: LineAndColumn
   ): nodes.NewMethod = {
-    val methodNode = nodeBuilder.methodNode(name, fullName, absFileName, lineAndColumn)
+    val methodNode = nodeBuilder.methodNode(name, fullName, relFileName, lineAndColumn)
     edgeBuilder.astEdge(methodNode, contextStack.astParent, contextStack.order.getAndInc)
 
     val blockNode = nodeBuilder.blockNode("", lineAndColumn)
@@ -377,7 +373,7 @@ class PythonAstVisitor(
     // For every method we create a corresponding TYPE and TYPE_DECL and
     // a binding for the method into TYPE_DECL.
     val typeNode     = nodeBuilder.typeNode(name, fullName)
-    val typeDeclNode = nodeBuilder.typeDeclNode(name, fullName, absFileName, Seq(Constants.ANY), lineAndColumn)
+    val typeDeclNode = nodeBuilder.typeDeclNode(name, fullName, relFileName, Seq(Constants.ANY), lineAndColumn)
 
     // For every method that is a module, the local variables can be imported by other modules. This behaviour is
     // much like fields so they are to be linked as fields to this method type
@@ -407,7 +403,7 @@ class PythonAstVisitor(
       nodeBuilder.typeDeclNode(
         metaTypeDeclName,
         metaTypeDeclFullName,
-        absFileName,
+        relFileName,
         Seq(Constants.ANY),
         lineAndColOf(classDef)
       )
@@ -439,7 +435,7 @@ class PythonAstVisitor(
       nodeBuilder.typeDeclNode(
         instanceTypeDeclName,
         instanceTypeDeclFullName,
-        absFileName,
+        relFileName,
         inheritsFrom,
         lineAndColOf(classDef)
       )

diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/Py2CpgTestContext.scala
@@ -25,10 +25,10 @@ class Py2CpgTestContext {
     if (buildResult.nonEmpty) {
       throw new RuntimeException("Not allowed to add sources after buildCpg() was called.")
     }
-    if (codeAndFile.exists(_.absFileName == file)) {
+    if (codeAndFile.exists(_.relFileName == file)) {
       throw new RuntimeException(s"Add more than one source under file name $file.")
     }
-    codeAndFile.append(Py2Cpg.InputPair(code, absTestFilePath + file, file))
+    codeAndFile.append(Py2Cpg.InputPair(code, file))
     this
   }
 

diff --git a/...n-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala b/...n-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/cpg/FunctionDefCpgTests.scala
@@ -15,7 +15,7 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers {
       val methodNode = cpg.method.fullName("test.py:<module>.func").head
       methodNode.name shouldBe "func"
       methodNode.fullName shouldBe "test.py:<module>.func"
-      methodNode.filename shouldBe "<absoluteTestPath>/test.py"
+      methodNode.filename shouldBe "test.py"
       methodNode.isExternal shouldBe false
       methodNode.lineNumber shouldBe Some(1)
       methodNode.columnNumber shouldBe Some(1)
@@ -71,7 +71,7 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers {
 
       bindingTypeDecl.name shouldBe "func"
       bindingTypeDecl.fullName shouldBe "test.py:<module>.func"
-      bindingTypeDecl.filename shouldBe "<absoluteTestPath>/test.py"
+      bindingTypeDecl.filename shouldBe "test.py"
       bindingTypeDecl.lineNumber shouldBe Some(1)
       bindingTypeDecl.columnNumber shouldBe Some(1)
 

diff --git a/.../frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala b/.../frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala
@@ -459,7 +459,7 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) {
 
     "correctly determine that, despite being unable to resolve the correct method full name, that it is an internal method" in {
       val Some(selfFindFound) = cpg.typeDecl(".*InstallationsDAO.*").ast.isCall.name("find_one").headOption: @unchecked
-      selfFindFound.callee.isExternal.toSeq shouldBe Seq(true, false)
+      selfFindFound.callee.isExternal.toSeq shouldBe Seq(true, true)
     }
   }