Python: Dataflow, Captured variables

python/ql/test/experimental/dataflow/coverage/argumentPassing.py

@@ -112,7 +112,7 @@ def with_multiple_kw_args(a, b, c):
     SINK3(c)
 
 
-@expects(9)
+@expects(12)
 def test_multiple_kw_args():
     with_multiple_kw_args(b=arg2, c=arg3, a=arg1)
     with_multiple_kw_args(arg1, *(arg2,), arg3)

python/ql/test/experimental/dataflow/coverage/validTest.py → python/ql/test/experimental/dataflow/validTest.py

@@ -50,6 +50,9 @@ def check_tests_valid(testFile):
 
 
 if __name__ == "__main__":
-    check_tests_valid("classes")
-    check_tests_valid("test")
-    check_tests_valid("argumentPassing")
+    check_tests_valid("coverage.classes")
+    check_tests_valid("coverage.test")
+    check_tests_valid("coverage.argumentPassing")
+    check_tests_valid("variable-capture.in")
+    check_tests_valid("variable-capture.nonlocal")
+    check_tests_valid("variable-capture.dict")

python/ql/test/experimental/dataflow/variable-capture/CaptureTest.ql

@@ -0,0 +1,21 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+import TestUtilities.InlineExpectationsTest
+import experimental.dataflow.testConfig
+
+class CaptureTest extends InlineExpectationsTest {
+  CaptureTest() { this = "CaptureTest" }
+
+  override string getARelevantTag() { result = "captured" }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(DataFlow::Node source, DataFlow::Node sink |
+      exists(TestConfiguration cfg | cfg.hasFlow(source, sink))
+    |
+      location = sink.getLocation() and
+      tag = "captured" and
+      value = "" and
+      element = sink.toString()
+    )
+  }
+}

python/ql/test/experimental/dataflow/variable-capture/dict.py

@@ -0,0 +1,90 @@
+# All functions starting with "test_" should run and execute `print("OK")` exactly once.
+# This can be checked by running validTest.py.
+
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import *
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+
+def Out():
+    sinkO1 = { "x": "" }
+    def captureOut1():
+        sinkO1["x"] = SOURCE
+    captureOut1()
+    SINK(sinkO1["x"]) #$ MISSING:captured
+
+    sinkO2 = { "x": "" }
+    def captureOut2():
+        def m():
+            sinkO2["x"] = SOURCE
+        m()
+    captureOut2()
+    SINK(sinkO2["x"]) #$ MISSING:captured
+
+    nonSink0 = { "x": "" }
+    def captureOut1NotCalled():
+        nonSink0["x"] = SOURCE
+    SINK_F(nonSink0["x"])
+
+    def captureOut2NotCalled():
+        def m():
+            nonSink0["x"] = SOURCE
+    captureOut2NotCalled()
+    SINK_F(nonSink0["x"])
+
+@expects(4)
+def test_Out():
+    Out()
+
+def Through(tainted):
+    sinkO1 = { "x": "" }
+    def captureOut1():
+        sinkO1["x"] = tainted
+    captureOut1()
+    SINK(sinkO1["x"]) #$ MISSING:captured
+
+    sinkO2 = { "x": "" }
+    def captureOut2():
+        def m():
+            sinkO2["x"] = tainted
+        m()
+    captureOut2()
+    SINK(sinkO2["x"]) #$ MISSING:captured
+
+    nonSink0 = { "x": "" }
+    def captureOut1NotCalled():
+        nonSink0["x"] = tainted
+    SINK_F(nonSink0["x"])
+
+    def captureOut2NotCalled():
+        def m():
+            nonSink0["x"] = tainted
+    captureOut2NotCalled()
+    SINK_F(nonSink0["x"])
+
+@expects(4)
+def test_Through():
+    Through(SOURCE)

python/ql/test/experimental/dataflow/variable-capture/in.py

@@ -0,0 +1,93 @@
+# All functions starting with "test_" should run and execute `print("OK")` exactly once.
+# This can be checked by running validTest.py.
+
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import *
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+
+def inParam(tainted):
+    def captureIn1():
+        sinkI1 = tainted
+        SINK(sinkI1) #$ MISSING:captured
+    captureIn1()
+
+    def captureIn2():
+        def m():
+            sinkI2 = tainted
+            SINK(sinkI2) #$ MISSING:captured
+        m()
+    captureIn2()
+
+    captureIn3 = lambda arg: SINK(tainted)
+    captureIn3("")
+
+    def captureIn1NotCalled():
+        nonSink0 = tainted
+        SINK_F(nonSink0)
+
+    def captureIn2NotCalled():
+        def m():
+            nonSink0 = tainted
+            SINK_F(nonSink0)
+    captureIn2NotCalled()
+
+@expects(3)
+def test_inParam():
+    inParam(SOURCE)
+
+def inLocal():
+    tainted = SOURCE
+
+    def captureIn1():
+        sinkI1 = tainted
+        SINK(sinkI1) #$ MISSING:captured
+    captureIn1()
+
+    def captureIn2():
+        def m():
+            sinkI2 = tainted
+            SINK(sinkI2) #$ MISSING:captured
+        m()
+    captureIn2()
+
+    captureIn3 = lambda arg: SINK(tainted)
+    captureIn3("")
+
+    def captureIn1NotCalled():
+        nonSink0 = tainted
+        SINK_F(nonSink0)
+
+    def captureIn2NotCalled():
+        def m():
+            nonSink0 = tainted
+            SINK_F(nonSink0)
+    captureIn2NotCalled()
+
+@expects(3)
+def test_inLocal():
+    inLocal()
+

python/ql/test/experimental/dataflow/variable-capture/nonlocal.py

@@ -0,0 +1,98 @@
+# All functions starting with "test_" should run and execute `print("OK")` exactly once.
+# This can be checked by running validTest.py.
+
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname((__file__))))
+from testlib import *
+
+# These are defined so that we can evaluate the test code.
+NONSOURCE = "not a source"
+SOURCE = "source"
+
+def is_source(x):
+    return x == "source" or x == b"source" or x == 42 or x == 42.0 or x == 42j
+
+
+def SINK(x):
+    if is_source(x):
+        print("OK")
+    else:
+        print("Unexpected flow", x)
+
+
+def SINK_F(x):
+    if is_source(x):
+        print("Unexpected flow", x)
+    else:
+        print("OK")
+
+
+def Out():
+    sinkO1 = ""
+    def captureOut1():
+        nonlocal sinkO1
+        sinkO1 = "source"
+    captureOut1()
+    SINK(sinkO1) #$ MISSING:captured
+
+    sinkO2 = ""
+    def captureOut2():
+        def m():
+            nonlocal sinkO2
+            sinkO2 = "source"
+        m()
+    captureOut2()
+    SINK(sinkO2) #$ MISSING:captured
+
+    nonSink0 = ""
+    def captureOut1NotCalled():
+        nonlocal nonSink0
+        nonSink0 = "source"
+    SINK_F(nonSink0)
+
+    def captureOut2NotCalled():
+        def m():
+            nonlocal nonSink0
+            nonSink0 = "source"
+    captureOut2NotCalled()
+    SINK_F(nonSink0)
+
+@expects(4)
+def test_Out():
+    Out()
+
+def Through(tainted):
+    sinkO1 = ""
+    def captureOut1():
+        nonlocal sinkO1
+        sinkO1 = tainted
+    captureOut1()
+    SINK(sinkO1) #$ MISSING:captured
+
+    sinkO2 = ""
+    def captureOut2():
+        def m():
+            nonlocal sinkO2
+            sinkO2 = tainted
+        m()
+    captureOut2()
+    SINK(sinkO2) #$ MISSING:captured
+
+    nonSink0 = ""
+    def captureOut1NotCalled():
+        nonlocal nonSink0
+        nonSink0 = tainted
+    SINK_F(nonSink0)
+
+    def captureOut2NotCalled():
+        def m():
+            nonlocal nonSink0
+            nonSink0 = tainted
+    captureOut2NotCalled()
+    SINK_F(nonSink0)
+
+@expects(4)
+def test_Through():
+    Through(SOURCE)