Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Many annotation fixes, pydetector dependency upgraded #69

Merged
merged 1 commit into from
Aug 11, 2017
Merged

Many annotation fixes, pydetector dependency upgraded #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 186 additions & 0 deletions ANNOTATION.md
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Dockerfile.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ MAINTAINER source{d}

ARG DEVDEPS=native/dev_deps
ARG CONTAINER_DEVDEPS=/tmp/dev_deps
ARG PYDETECTOR_VER=0.11.3
ARG PYDETECTOR_VER=0.12.0

RUN apk add --no-cache --update python python3 py-pip py2-pip git

Expand Down
27 changes: 15 additions & 12 deletions driver/normalizer/annotation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,17 +207,19 @@ var AnnotationRules = On(Any).Self(
// TODO: check what Constant nodes are generated in the python AST and improve this
On(HasInternalType(pyast.Constant)).Roles(SimpleIdentifier, Expression),
On(HasInternalType(pyast.Try)).Roles(Try, Statement).Children(
On(HasInternalRole("body")).Roles(TryBody),
On(HasInternalRole("finalbody")).Roles(TryFinally),
On(HasInternalRole("handlers")).Roles(TryCatch),
On(HasInternalRole("orelse")).Roles(IfElse),
On(HasInternalType("Try.body")).Roles(TryBody),
On(HasInternalType("Try.finalbody")).Roles(TryFinally),
On(HasInternalType("Try.handlers")).Roles(TryCatch),
On(HasInternalType("Try.orelse")).Roles(IfElse),
),
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch, Statement), // py2
On(HasInternalType(pyast.ExceptHandler)).Roles(TryCatch, Statement), // py3
On(HasInternalType(pyast.TryFinally)).Roles(TryFinally, Statement),
On(HasInternalType(pyast.Raise)).Roles(Throw, Statement),
// FIXME: review, add path for the body and items childs
On(HasInternalType(pyast.With)).Roles(BlockScope, Statement),
On(HasInternalType("With.body")).Roles(BlockScope, Expression, Incomplete),
On(HasInternalType("With.items")).Roles(SimpleIdentifier, Expression, Incomplete),
On(HasInternalType(pyast.AsyncWith)).Roles(BlockScope, Statement, Incomplete),
On(HasInternalType(pyast.Withitem)).Roles(SimpleIdentifier, Incomplete),
On(HasInternalType(pyast.Return)).Roles(Return, Statement),
Expand All @@ -236,15 +238,15 @@ var AnnotationRules = On(Any).Self(
// and SDK feature to mix lists (also needed for default and keyword arguments and
// boolean operators).
// "If that sounds awkward is because it is" (their words)
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression, Expression).Children(
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
),
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
On(HasInternalType(pyast.If)).Roles(If, Statement).Children(
On(HasInternalType("If.body")).Roles(IfBody),
On(HasInternalRole("test")).Roles(IfCondition),
On(HasInternalType("If.orelse")).Roles(IfElse),
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression, Expression).Children(
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
),
),
On(HasInternalType(pyast.IfExp)).Roles(If, Expression).Children(
// These are used on ifexpressions (a = 1 if x else 2)
Expand All @@ -263,7 +265,9 @@ var AnnotationRules = On(Any).Self(
On(HasInternalType(pyast.ClassDef)).Roles(TypeDeclaration, SimpleIdentifier, Statement).Children(
On(HasInternalType("ClassDef.body")).Roles(TypeDeclarationBody),
On(HasInternalType("ClassDef.bases")).Roles(TypeDeclarationBases),
),
On(HasInternalType("ClassDef.keywords")).Roles(Incomplete).Children(
On(HasInternalType(pyast.Keyword)).Roles(SimpleIdentifier, Incomplete),
)),

On(HasInternalType(pyast.For)).Roles(ForEach, Statement).Children(
On(HasInternalType("For.body")).Roles(ForBody),
Expand Down Expand Up @@ -327,12 +331,11 @@ var AnnotationRules = On(Any).Self(
// FIXME: see the comment on IfCondition above
On(HasInternalType(pyast.Compare)).Roles(IfCondition, BinaryExpression).Children(
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
),
),
On(HasInternalType(pyast.ListComp)).Roles(ListLiteral, Expression, Incomplete),
On(HasInternalType(pyast.SetComp)).Roles(MapLiteral, Expression, Incomplete),
On(HasInternalType(pyast.DictComp)).Roles(MapLiteral, Expression, Incomplete),
On(HasInternalType(pyast.SetComp)).Roles(SetLiteral, Expression, Incomplete),

On(HasInternalType(pyast.Delete)).Roles(Statement, Incomplete),
Expand Down
8 changes: 1 addition & 7 deletions driver/normalizer/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,7 @@ var ToNoder = &native.ObjectToNoder{
"For": {"body": true, "orelse": true},
"AsyncFor": {"body": true, "orelse": true},
"While": {"body": true, "orelse": true},
// FIXME: check if promotion is needed in this case
"Compare": {"comparators": true, "ops": true},
"Import": {"names": true},
"ImportFrom": {"names": true},
"Compare": {"comparators": true, "ops": true},
// FIXME: check call.keywords
//"Call" : { "args": true, "keywords": true},
"With": {"body": true, "items": true},
Expand All @@ -73,9 +70,6 @@ var ToNoder = &native.ObjectToNoder{
"Try": {"body": true, "orelse": true, "finalbody": true, "handlers": true},
"Raise": {"args": true},
"ClassDef": {"body": true, "bases": true, "decorator_list": true, "keywords": true},
"ListComp": {"generators": true},
"ListComp.generators": {"ifs": true},
"ListComp.generators.ifs": {"comparators": true, "ops": true},
},
PromotedPropertyStrings: map[string]map[string]bool{
"alias": {"asname": true},
Expand Down
2 changes: 1 addition & 1 deletion native/python_package/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
pydetector-bblfsh==0.11.3
pydetector-bblfsh==0.12.0
-e git+git://github.com/python/mypy.git@0bb2d1680e8b9522108b38d203cb73021a617e64#egg=mypy-lang
typed-ast==1.0.1
2 changes: 1 addition & 1 deletion native/python_package/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
]
},
install_requires=[
"pydetector-bblfsh==0.11.3"
"pydetector-bblfsh==0.12.0"
],
classifiers=[
"Development Status :: 4 - Beta",
Expand Down
40 changes: 20 additions & 20 deletions tests/booleanop.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Module {
. . . }
. . . Children: {
. . . . 0: Compare {
. . . . . Roles: Unannotated
. . . . . Roles: BinaryExpression,Expression
. . . . . StartPosition: {
. . . . . . Offset: 0
. . . . . . Line: 1
Expand All @@ -47,7 +47,7 @@ Module {
. . . . . }
. . . . . Children: {
. . . . . . 0: Compare.comparators {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionRight
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand All @@ -72,7 +72,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 1: NumLiteral {
. . . . . . . Roles: NumberLiteral,Expression
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
. . . . . . . TOKEN "1"
. . . . . . . StartPosition: {
. . . . . . . . Offset: 0
Expand All @@ -90,7 +90,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 2: Compare.ops {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionOp
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down Expand Up @@ -132,7 +132,7 @@ Module {
. . . }
. . . Children: {
. . . . 0: Compare {
. . . . . Roles: Unannotated
. . . . . Roles: BinaryExpression,Expression
. . . . . StartPosition: {
. . . . . . Offset: 7
. . . . . . Line: 2
Expand All @@ -148,7 +148,7 @@ Module {
. . . . . }
. . . . . Children: {
. . . . . . 0: Compare.comparators {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionRight
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand All @@ -173,7 +173,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 1: NumLiteral {
. . . . . . . Roles: NumberLiteral,Expression
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
. . . . . . . TOKEN "1"
. . . . . . . StartPosition: {
. . . . . . . . Offset: 7
Expand All @@ -191,7 +191,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 2: Compare.ops {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionOp
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down Expand Up @@ -304,7 +304,7 @@ Module {
. . . }
. . . Children: {
. . . . 0: Compare {
. . . . . Roles: Unannotated
. . . . . Roles: BinaryExpression,Expression
. . . . . StartPosition: {
. . . . . . Offset: 20
. . . . . . Line: 4
Expand All @@ -320,7 +320,7 @@ Module {
. . . . . }
. . . . . Children: {
. . . . . . 0: Compare.comparators {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionRight
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down Expand Up @@ -362,7 +362,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 1: NumLiteral {
. . . . . . . Roles: NumberLiteral,Expression
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
. . . . . . . TOKEN "1"
. . . . . . . StartPosition: {
. . . . . . . . Offset: 20
Expand All @@ -380,7 +380,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 2: Compare.ops {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionOp
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down Expand Up @@ -436,7 +436,7 @@ Module {
. . . }
. . . Children: {
. . . . 0: Compare {
. . . . . Roles: Unannotated
. . . . . Roles: BinaryExpression,Expression
. . . . . StartPosition: {
. . . . . . Offset: 32
. . . . . . Line: 5
Expand All @@ -452,7 +452,7 @@ Module {
. . . . . }
. . . . . Children: {
. . . . . . 0: Compare.comparators {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionRight
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down Expand Up @@ -494,7 +494,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 1: NumLiteral {
. . . . . . . Roles: NumberLiteral,Expression
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
. . . . . . . TOKEN "1"
. . . . . . . StartPosition: {
. . . . . . . . Offset: 32
Expand All @@ -512,7 +512,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 2: Compare.ops {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionOp
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down Expand Up @@ -568,7 +568,7 @@ Module {
. . . }
. . . Children: {
. . . . 0: Compare {
. . . . . Roles: Unannotated
. . . . . Roles: BinaryExpression,Expression
. . . . . StartPosition: {
. . . . . . Offset: 42
. . . . . . Line: 6
Expand All @@ -584,7 +584,7 @@ Module {
. . . . . }
. . . . . Children: {
. . . . . . 0: Compare.comparators {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionRight
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down Expand Up @@ -626,7 +626,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 1: NumLiteral {
. . . . . . . Roles: NumberLiteral,Expression
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
. . . . . . . TOKEN "1"
. . . . . . . StartPosition: {
. . . . . . . . Offset: 42
Expand All @@ -644,7 +644,7 @@ Module {
. . . . . . . }
. . . . . . }
. . . . . . 2: Compare.ops {
. . . . . . . Roles: Unannotated
. . . . . . . Roles: BinaryExpressionOp
. . . . . . . Properties: {
. . . . . . . . promotedPropertyList: true
. . . . . . . }
Expand Down
12 changes: 7 additions & 5 deletions tests/classdef.py.native
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,13 +160,15 @@
{
"ast_type": "Pass",
"col_offset": 9,
"end_col_offset": 12,
"end_lineno": 7,
"lineno": 7
}
],
"col_offset": 9,
"decorator_list": [],
"end_col_offset": 25,
"end_lineno": 6,
"end_col_offset": 12,
"end_lineno": 7,
"lineno": 6,
"name": "method",
"returns": null
Expand Down Expand Up @@ -280,7 +282,7 @@
"kw_defaults": [],
"kwarg": null,
"kwonlyargs": [],
"lineno": 13,
"lineno": 14,
"vararg": null
},
"ast_type": "FunctionDef",
Expand Down Expand Up @@ -322,7 +324,7 @@
}
}
],
"col_offset": 5,
"col_offset": 9,
"decorator_list": [
{
"ast_type": "Attribute",
Expand All @@ -345,7 +347,7 @@
],
"end_col_offset": 22,
"end_lineno": 15,
"lineno": 13,
"lineno": 14,
"name": "a",
"returns": null
}
Expand Down
21 changes: 13 additions & 8 deletions tests/classdef.py.uast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,9 +233,9 @@ Module {
. . . . . . . . Col: 9
. . . . . . . }
. . . . . . . EndPosition: {
. . . . . . . . Offset: 97
. . . . . . . . Line: 6
. . . . . . . . Col: 25
. . . . . . . . Offset: 112
. . . . . . . . Line: 7
. . . . . . . . Col: 12
. . . . . . . }
. . . . . . . Properties: {
. . . . . . . . returns: <nil>
Expand Down Expand Up @@ -344,6 +344,11 @@ Module {
. . . . . . . . . . . . Line: 7
. . . . . . . . . . . . Col: 9
. . . . . . . . . . . }
. . . . . . . . . . . EndPosition: {
. . . . . . . . . . . . Offset: 112
. . . . . . . . . . . . Line: 7
. . . . . . . . . . . . Col: 12
. . . . . . . . . . . }
. . . . . . . . . . }
. . . . . . . . . }
. . . . . . . . }
Expand Down Expand Up @@ -504,9 +509,9 @@ Module {
. . . . . . . Roles: FunctionDeclaration,FunctionDeclarationName,SimpleIdentifier
. . . . . . . TOKEN "a"
. . . . . . . StartPosition: {
. . . . . . . . Offset: 170
. . . . . . . . Line: 13
. . . . . . . . Col: 5
. . . . . . . . Offset: 188
. . . . . . . . Line: 14
. . . . . . . . Col: 9
. . . . . . . }
. . . . . . . EndPosition: {
. . . . . . . . Offset: 218
Expand All @@ -520,8 +525,8 @@ Module {
. . . . . . . . 0: arguments {
. . . . . . . . . Roles: FunctionDeclarationArgument,Incomplete
. . . . . . . . . StartPosition: {
. . . . . . . . . . Offset: 177
. . . . . . . . . . Line: 13
. . . . . . . . . . Offset: 191
. . . . . . . . . . Line: 14
. . . . . . . . . . Col: 12
. . . . . . . . . }
. . . . . . . . . EndPosition: {
Expand Down
Loading