Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make the filename property in pysrc2cpg relative again. #3544

Merged
merged 1 commit into from
Aug 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,13 @@ class CodeToCpg(cpg: Cpg, inputProvider: Iterable[InputProvider], schemaValidati
val lineBreakCorrectedCode = inputPair.content.replace("\r\n", "\n").replace("\r", "\n")
val astRoot = parser.parse(lineBreakCorrectedCode)
val nodeToCode = new NodeToCode(lineBreakCorrectedCode)
val astVisitor = new PythonAstVisitor(inputPair.absFileName, inputPair.relFileName, nodeToCode, PythonV2AndV3)(
schemaValidationMode
)
val astVisitor = new PythonAstVisitor(inputPair.relFileName, nodeToCode, PythonV2AndV3)(schemaValidationMode)
astVisitor.convert(astRoot)

diffGraph.absorb(astVisitor.getDiffGraph)
} catch {
case exception: Throwable =>
logger.warn(s"Failed to convert file ${inputPair.absFileName}", exception)
logger.warn(s"Failed to convert file ${inputPair.relFileName}", exception)
Iterator.empty
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import overflowdb.BatchedUpdate
import overflowdb.BatchedUpdate.DiffGraphBuilder

object Py2Cpg {
case class InputPair(content: String, absFileName: String, relFileName: String)
case class InputPair(content: String, relFileName: String)
type InputProvider = () => InputPair
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class Py2CpgOnFileSystem extends X2CpgFrontend[Py2CpgOnFileSystemConfig] {
val inputProviders = inputFiles.map { inputFile => () =>
{
val content = IOUtils.readLinesInFile(inputFile).mkString("\n")
Py2Cpg.InputPair(content, inputFile.toString, Paths.get(config.inputPath).relativize(inputFile).toString)
Py2Cpg.InputPair(content, Paths.get(config.inputPath).relativize(inputFile).toString)
}
}
val py2Cpg = new Py2Cpg(inputProviders, cpg, config.inputPath, config.requirementsTxt, config.schemaValidation)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,9 @@ object PythonV2 extends PythonVersion
object PythonV3 extends PythonVersion
object PythonV2AndV3 extends PythonVersion

class PythonAstVisitor(
absFileName: String,
relFileName: String,
protected val nodeToCode: NodeToCode,
version: PythonVersion
)(implicit withSchemaValidation: ValidationMode)
extends PythonAstVisitorHelpers {
class PythonAstVisitor(relFileName: String, protected val nodeToCode: NodeToCode, version: PythonVersion)(implicit
withSchemaValidation: ValidationMode
) extends PythonAstVisitorHelpers {

private val diffGraph = new DiffGraphBuilder()
protected val nodeBuilder = new NodeBuilder(diffGraph)
Expand Down Expand Up @@ -76,12 +72,12 @@ class PythonAstVisitor(
module.accept(memOpCalculator)
memOpMap = memOpCalculator.astNodeToMemOp

val fileNode = nodeBuilder.fileNode(absFileName)
val fileNode = nodeBuilder.fileNode(relFileName)
val namespaceBlockNode =
nodeBuilder.namespaceBlockNode(
Constants.GLOBAL_NAMESPACE,
relFileName + ":" + Constants.GLOBAL_NAMESPACE,
absFileName
relFileName
)
edgeBuilder.astEdge(namespaceBlockNode, fileNode, 1)
contextStack.setFileNamespaceBlock(namespaceBlockNode)
Expand Down Expand Up @@ -346,7 +342,7 @@ class PythonAstVisitor(
returnTypeHint: Option[String],
lineAndColumn: LineAndColumn
): nodes.NewMethod = {
val methodNode = nodeBuilder.methodNode(name, fullName, absFileName, lineAndColumn)
val methodNode = nodeBuilder.methodNode(name, fullName, relFileName, lineAndColumn)
edgeBuilder.astEdge(methodNode, contextStack.astParent, contextStack.order.getAndInc)

val blockNode = nodeBuilder.blockNode("", lineAndColumn)
Expand Down Expand Up @@ -377,7 +373,7 @@ class PythonAstVisitor(
// For every method we create a corresponding TYPE and TYPE_DECL and
// a binding for the method into TYPE_DECL.
val typeNode = nodeBuilder.typeNode(name, fullName)
val typeDeclNode = nodeBuilder.typeDeclNode(name, fullName, absFileName, Seq(Constants.ANY), lineAndColumn)
val typeDeclNode = nodeBuilder.typeDeclNode(name, fullName, relFileName, Seq(Constants.ANY), lineAndColumn)

// For every method that is a module, the local variables can be imported by other modules. This behaviour is
// much like fields so they are to be linked as fields to this method type
Expand Down Expand Up @@ -407,7 +403,7 @@ class PythonAstVisitor(
nodeBuilder.typeDeclNode(
metaTypeDeclName,
metaTypeDeclFullName,
absFileName,
relFileName,
Seq(Constants.ANY),
lineAndColOf(classDef)
)
Expand Down Expand Up @@ -439,7 +435,7 @@ class PythonAstVisitor(
nodeBuilder.typeDeclNode(
instanceTypeDeclName,
instanceTypeDeclFullName,
absFileName,
relFileName,
inheritsFrom,
lineAndColOf(classDef)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ class Py2CpgTestContext {
if (buildResult.nonEmpty) {
throw new RuntimeException("Not allowed to add sources after buildCpg() was called.")
}
if (codeAndFile.exists(_.absFileName == file)) {
if (codeAndFile.exists(_.relFileName == file)) {
throw new RuntimeException(s"Add more than one source under file name $file.")
}
codeAndFile.append(Py2Cpg.InputPair(code, absTestFilePath + file, file))
codeAndFile.append(Py2Cpg.InputPair(code, file))
this
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers {
val methodNode = cpg.method.fullName("test.py:<module>.func").head
methodNode.name shouldBe "func"
methodNode.fullName shouldBe "test.py:<module>.func"
methodNode.filename shouldBe "<absoluteTestPath>/test.py"
methodNode.filename shouldBe "test.py"
methodNode.isExternal shouldBe false
methodNode.lineNumber shouldBe Some(1)
methodNode.columnNumber shouldBe Some(1)
Expand Down Expand Up @@ -71,7 +71,7 @@ class FunctionDefCpgTests extends AnyFreeSpec with Matchers {

bindingTypeDecl.name shouldBe "func"
bindingTypeDecl.fullName shouldBe "test.py:<module>.func"
bindingTypeDecl.filename shouldBe "<absoluteTestPath>/test.py"
bindingTypeDecl.filename shouldBe "test.py"
bindingTypeDecl.lineNumber shouldBe Some(1)
bindingTypeDecl.columnNumber shouldBe Some(1)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,7 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) {

"correctly determine that, despite being unable to resolve the correct method full name, that it is an internal method" in {
val Some(selfFindFound) = cpg.typeDecl(".*InstallationsDAO.*").ast.isCall.name("find_one").headOption: @unchecked
selfFindFound.callee.isExternal.toSeq shouldBe Seq(true, false)
selfFindFound.callee.isExternal.toSeq shouldBe Seq(true, true)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This contradicts the description of the test case, though I wonder what the cause of this is - perhaps import resolution is confused.

}
}

Expand Down