Skip to content

Commit 7a510f5

Browse files
taus-semmletaus-semmle
authored
Merge pull request github#1609 from markshannon/python-better-points-to-extensions
Python points-to: Remove negative recursion when using legacy points-to extensions
2 parents 3f46492 + 3343f6b commit 7a510f5

File tree

19 files changed

+90
-44
lines changed

19 files changed

+90
-44
lines changed

‎python/ql/src/semmle/python/objects/Callables.qll

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
156156
function = this and offset = 0
157157
}
158158

159+
override predicate useOriginAsLegacyObject() { none() }
160+
159161
}
160162

161163

@@ -277,6 +279,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
277279
function = this and offset = 0
278280
}
279281

282+
override predicate useOriginAsLegacyObject() { none() }
283+
280284
}
281285

282286
/** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`.
@@ -367,6 +371,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
367371
function = this and offset = 0
368372
}
369373

374+
override predicate useOriginAsLegacyObject() { none() }
375+
370376
}
371377

372378
/** Class representing bound-methods.
@@ -453,6 +459,8 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
453459
function = this.getFunction() and offset = 1
454460
}
455461

462+
override predicate useOriginAsLegacyObject() { any() }
463+
456464
}
457465

458466

‎python/ql/src/semmle/python/objects/Classes.qll

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ abstract class ClassObjectInternal extends ObjectInternal {
8989
}
9090

9191
override predicate subscriptUnknown() { none() }
92+
93+
override predicate useOriginAsLegacyObject() { none() }
94+
9295
}
9396

9497
/** Class representing Python source classes */

‎python/ql/src/semmle/python/objects/Constants.qll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
6969

7070
override string getName() { none() }
7171

72+
override predicate useOriginAsLegacyObject() { none() }
73+
7274
}
7375

7476
private abstract class BooleanObjectInternal extends ConstantObjectInternal {

‎python/ql/src/semmle/python/objects/Descriptors.qll

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ class PropertyInternal extends ObjectInternal, TProperty {
9191
)
9292
}
9393

94+
override predicate useOriginAsLegacyObject() { none() }
9495
}
9596

9697
/** A class representing classmethods in Python */
@@ -176,6 +177,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
176177
result = this.getFunction().getName()
177178
}
178179

180+
override predicate useOriginAsLegacyObject() { none() }
181+
179182
}
180183

181184
class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
@@ -247,4 +250,6 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
247250
result = this.getFunction().getName()
248251
}
249252

253+
override predicate useOriginAsLegacyObject() { none() }
254+
250255
}

‎python/ql/src/semmle/python/objects/Instances.qll

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,8 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
160160
)
161161
}
162162

163+
override predicate useOriginAsLegacyObject() { none() }
164+
163165
}
164166

165167
/** A class representing context-free instances represented by `self` in the source code
@@ -262,6 +264,8 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
262264
this.getClass().attribute("__init__", init, _)
263265
}
264266

267+
override predicate useOriginAsLegacyObject() { none() }
268+
265269
}
266270

267271
/** A class representing a value that has a known class, but no other information */
@@ -366,6 +370,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
366370

367371
override string getName() { none() }
368372

373+
override predicate useOriginAsLegacyObject() { any() }
374+
369375
}
370376

371377
private int lengthFromClass(ClassObjectInternal cls) {
@@ -472,5 +478,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
472478

473479
override string getName() { none() }
474480

481+
override predicate useOriginAsLegacyObject() { any() }
482+
475483
}
476484

‎python/ql/src/semmle/python/objects/Modules.qll

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ abstract class ModuleObjectInternal extends ObjectInternal {
5252
any(PackageObjectInternal package).getInitModule() = this
5353
}
5454

55+
override predicate useOriginAsLegacyObject() { none() }
56+
5557
}
5658

5759
/** A class representing built-in modules */
@@ -308,10 +310,6 @@ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule {
308310
none()
309311
}
310312

311-
override predicate isMissing() {
312-
any()
313-
}
314-
315313
}
316314

317315
/** A class representing an attribute of a missing module. */
@@ -397,12 +395,10 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
397395

398396
override predicate subscriptUnknown() { any() }
399397

400-
override predicate isMissing() {
401-
any()
402-
}
403-
404398
/* We know what this is called, but not its innate name */
405399
override string getName() { none() }
406400

401+
override predicate useOriginAsLegacyObject() { none() }
402+
407403
}
408404

‎python/ql/src/semmle/python/objects/ObjectAPI.qll

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,14 +78,6 @@ class Value extends TObject {
7878
this.(ObjectInternal).isBuiltin()
7979
}
8080

81-
/** Holds if this value represents an entity that is inferred to exist,
82-
* but missing from the database.
83-
* Most commonly, this is a module that is imported, but wasn't present during extraction.
84-
*/
85-
predicate isMissing() {
86-
this.(ObjectInternal).isMissing()
87-
}
88-
8981
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
9082
this.(ObjectInternal).getOrigin().getLocation().hasLocationInfo(filepath, bl, bc, el, ec)
9183
or

‎python/ql/src/semmle/python/objects/ObjectInternal.qll

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -155,11 +155,12 @@ class ObjectInternal extends TObject {
155155
*/
156156
predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
157157

158-
/** Holds if this 'object' represents an entity that is inferred to exist
159-
* but is missing from the database */
160-
predicate isMissing() {
161-
none()
162-
}
158+
/** Holds if this 'object' represents an entity that should be exposed to the legacy points_to API
159+
* This should hold for almost all objects that do not have an underlying DB object representing their source,
160+
* for example `super` objects and bound-method. This should not hold for objects that are inferred to exists by
161+
* an import statements or the like, but which aren't in the database. */
162+
/* This predicate can be removed when the legacy points_to API is removed. */
163+
abstract predicate useOriginAsLegacyObject();
163164

164165
/** Gets the name of this of this object if it has a meaningful name.
165166
* Note that the name of an object is not necessarily the name by which it is called
@@ -249,6 +250,9 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
249250
override string getName() {
250251
result = this.getBuiltin().getName()
251252
}
253+
254+
override predicate useOriginAsLegacyObject() { none() }
255+
252256
}
253257

254258

@@ -326,6 +330,8 @@ class UnknownInternal extends ObjectInternal, TUnknown {
326330

327331
override string getName() { none() }
328332

333+
override predicate useOriginAsLegacyObject() { none() }
334+
329335
}
330336

331337
class UndefinedInternal extends ObjectInternal, TUndefined {
@@ -404,6 +410,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
404410

405411
override string getName() { none() }
406412

413+
override predicate useOriginAsLegacyObject() { none() }
414+
407415
}
408416

409417
module ObjectInternal {
@@ -498,6 +506,7 @@ module ObjectInternal {
498506
result.(BuiltinTupleObjectInternal).length() = 0
499507
}
500508

509+
501510
}
502511

503512
/** Helper for boolean predicates returning both `true` and `false` */

‎python/ql/src/semmle/python/objects/Sequences.qll

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
117117
result = count(int n | exists(b.getItem(n)))
118118
)
119119
}
120+
121+
override predicate useOriginAsLegacyObject() { none() }
122+
120123
}
121124

122125
/** A tuple declared by a tuple expression in the Python source code */
@@ -148,6 +151,8 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
148151
)
149152
}
150153

154+
override predicate useOriginAsLegacyObject() { none() }
155+
151156
}
152157

153158
/** A tuple created by a `*` parameter */
@@ -176,6 +181,9 @@ class VarargsTupleObjectInternal extends TVarargsTuple, TupleObjectInternal {
176181
override int length() {
177182
this = TVarargsTuple(_, _, _, result)
178183
}
184+
185+
override predicate useOriginAsLegacyObject() { any() }
186+
179187
}
180188

181189

@@ -256,4 +264,6 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
256264

257265
override predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
258266

267+
override predicate useOriginAsLegacyObject() { any() }
268+
259269
}

‎python/ql/src/semmle/python/pointsto/PointsTo.qll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ module PointsTo {
130130
PointsToInternal::pointsTo(f, context, value, origin) and
131131
cls = value.getClass().getSource() |
132132
obj = value.getSource() or
133-
not exists(value.getSource()) and not value.isMissing() and obj = origin
133+
value.useOriginAsLegacyObject() and obj = origin
134134
)
135135
or
136136
/* Backwards compatibility for *args and **kwargs */
@@ -145,7 +145,7 @@ module PointsTo {
145145
PointsToInternal::pointsTo(f.(DefinitionNode).getValue(), context, value, origin) and
146146
cls = value.getClass().getSource() |
147147
obj = value.getSource() or
148-
not exists(value.getSource()) and obj = origin
148+
value.useOriginAsLegacyObject() and obj = origin
149149
)
150150
}
151151

‎python/ql/src/semmle/python/regex.qll

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import python
2+
import semmle.python.objects.ObjectInternal
23

34
private predicate re_module_function(string name, int flags) {
45
name = "compile" and flags = 1 or
@@ -14,44 +15,42 @@ private predicate re_module_function(string name, int flags) {
1415
predicate used_as_regex(Expr s, string mode) {
1516
(s instanceof Bytes or s instanceof Unicode)
1617
and
17-
exists(ModuleObject re | re.getName() = "re" |
18+
exists(ModuleValue re | re.getName() = "re" |
1819
/* Call to re.xxx(regex, ... [mode]) */
1920
exists(CallNode call, string name |
2021
call.getArg(0).refersTo(_, _, s.getAFlowNode()) and
21-
call.getFunction().refersTo(re.attr(name)) |
22+
call.getFunction().pointsTo(re.attr(name)) |
2223
mode = "None"
2324
or
24-
exists(Object obj |
25+
exists(Value obj |
2526
mode = mode_from_mode_object(obj) |
2627
exists(int flags_arg |
2728
re_module_function(name, flags_arg) and
28-
call.getArg(flags_arg).refersTo(obj)
29+
call.getArg(flags_arg).pointsTo(obj)
2930
)
3031
or
31-
call.getArgByName("flags").refersTo(obj)
32+
call.getArgByName("flags").pointsTo(obj)
3233
)
3334
)
3435
)
3536
}
3637

37-
string mode_from_mode_object(Object obj) {
38+
string mode_from_mode_object(Value obj) {
3839
(
3940
result = "DEBUG" or result = "IGNORECASE" or result = "LOCALE" or
4041
result = "MULTILINE" or result = "DOTALL" or result = "UNICODE" or
4142
result = "VERBOSE"
4243
) and
43-
obj = ModuleObject::named("sre_constants").attr("SRE_FLAG_" + result)
44-
or
45-
exists(BinaryExpr be, Object sub | obj.getOrigin() = be |
46-
be.getOp() instanceof BitOr and
47-
be.getASubExpression().refersTo(sub) and
48-
result = mode_from_mode_object(sub)
44+
exists(int flag |
45+
flag = Value::named("sre_constants.SRE_FLAG_" + result).(ObjectInternal).intValue()
46+
and
47+
obj.(ObjectInternal).intValue().bitAnd(flag) = flag
4948
)
5049
}
5150

5251
/** A StrConst used as a regular expression */
5352
abstract class RegexString extends Expr {
54-
53+
5554
RegexString() {
5655
(this instanceof Bytes or this instanceof Unicode)
5756
}

‎python/ql/src/semmle/python/types/Extensions.qll

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ import python
1313
private import semmle.python.pointsto.PointsTo
1414
private import semmle.python.pointsto.PointsToContext
1515
private import semmle.python.objects.TObject
16-
private import semmle.python.objects.ObjectInternal
1716
private import semmle.python.web.HttpConstants
1817

18+
/* Make ObjectInternal visible to save extra imports in user code */
19+
import semmle.python.objects.ObjectInternal
20+
1921
abstract class PointsToExtension extends @py_flow_node {
2022

2123
string toString() { none() }

‎python/ql/test/3/library-tests/PointsTo/attributes/Test.expected

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
| 1 | ControlFlowNode for unicode_literals | ImportMember | 1 |
21
| 2 | ControlFlowNode for C | class C | 2 |
32
| 2 | ControlFlowNode for ClassExpr | class C | 2 |
43
| 2 | ControlFlowNode for object | builtin-class object | 2 |

‎python/ql/test/library-tests/PointsTo/extensions/Extend.expected

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
WARNING: Type CustomPointsToAttribute has been deprecated and may be removed in future (Extend.ql:26,35-58)
2-
WARNING: Type CustomPointsToObjectFact has been deprecated and may be removed in future (Extend.ql:41,32-56)
3-
WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (Extend.ql:8,28-52)
1+
WARNING: Predicate points_to has been deprecated and may be removed in future (Extend.ql:58,9-28)
2+
WARNING: Type CustomPointsToAttribute has been deprecated and may be removed in future (Extend.ql:27,35-58)
3+
WARNING: Type CustomPointsToObjectFact has been deprecated and may be removed in future (Extend.ql:42,32-56)
4+
WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (Extend.ql:9,28-52)
5+
WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (Extend.ql:55,38-62)
46
| test.py:4:1:4:3 | ControlFlowNode for one | int 1 |
57
| test.py:5:1:5:3 | ControlFlowNode for two | int 2 |
68
| test.py:8:1:8:1 | ControlFlowNode for IntegerLiteral | int 1 |

‎python/ql/test/library-tests/PointsTo/extensions/Extend.ql

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import python
44

5+
import semmle.python.pointsto.PointsTo
56
private import semmle.python.types.Extensions
67

78

@@ -50,6 +51,18 @@ class NoClassExtension extends CustomPointsToObjectFact {
5051

5152
}
5253

54+
/* Check that we can use old API without causing non-monotonic recursion */
55+
class RecurseIntoOldPointsTo extends CustomPointsToOriginFact {
56+
57+
RecurseIntoOldPointsTo() {
58+
PointsTo::points_to(this, _, unknownValue(), _, _)
59+
}
60+
61+
override predicate pointsTo(Object value, ClassObject cls) {
62+
value = unknownValue() and cls = theUnknownType()
63+
}
64+
}
65+
5366

5467
from ControlFlowNode f, Object o
5568
where f.getLocation().getFile().getBaseName() = "test.py" and f.refersTo(o)

‎python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,6 @@
9898
| Module pointsto_test | 76 | ControlFlowNode for sys | Module sys |
9999
| Module pointsto_test | 76 | ControlFlowNode for type | builtin-class type |
100100
| Module pointsto_test | 76 | ControlFlowNode for type() | builtin-class module |
101-
| Module pointsto_test | 77 | ControlFlowNode for unknown | ImportMember |
102101
| Module pointsto_test | 78 | ControlFlowNode for type | builtin-class type |
103102
| Module pointsto_test | 79 | ControlFlowNode for Dict | Dict |
104103
| Module pointsto_test | 79 | ControlFlowNode for Tuple | Tuple |

0 commit comments

Comments
 (0)