Prepare dataflow for local annotations

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatch.qll

@@ -1714,36 +1714,54 @@ private class SummaryPostUpdateNode extends FlowSummaryNode, PostUpdateNodeImpl
  * This is also known as the environment part of a closure.
  *
  * This is used for tracking flow through captured variables.
- *
- * TODO:
- * We might want a synthetic node here, but currently that incurs problems
- * with non-monotonic recursion, because of the use of `resolveCall` in the
- * char pred. This may be solvable by using
- * `CallGraphConstruction::Make` in stead of
- * `CallGraphConstruction::Simple::Make` appropriately.
  */
-class CapturedVariablesArgumentNode extends CfgNode, ArgumentNode {
+class CapturedVariablesArgumentNode extends Node, TSynthCapturedVariablesArgumentNode {
   CallNode callNode;
 
   CapturedVariablesArgumentNode() {
-    node = callNode.getFunction() and
-    exists(Function target | resolveCall(callNode, target, _) |
-      target = any(VariableCapture::CapturedVariable v).getACapturingScope()
+    exists(ControlFlowNode callable |
+      this = TSynthCapturedVariablesArgumentNode(callable) and callable = callNode.getFunction()
     )
   }
 
+  /** Gets the call node associated with this captured variables argument. */
+  CallNode getCallNode() { result = callNode }
+
+  override Location getLocation() { result = callNode.getLocation() }
+
   override string toString() { result = "Capturing closure argument" }
+}
 
+/** A captured variables argument node viewed as an argument node. Needed because `argumentOf` is a global predicate. */
+class CapturedVariablesArgumentNodeAsArgumentNode extends ArgumentNode instanceof CapturedVariablesArgumentNode
+{
   override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
-    callNode = call.getNode() and
+    this.(CapturedVariablesArgumentNode).getCallNode() = call.getNode() and
     pos.isLambdaSelf()
   }
 }
 
-/** A synthetic node representing the values of variables captured by a comprehension. */
-class SynthCompCapturedVariablesArgumentNode extends Node, TSynthCompCapturedVariablesArgumentNode,
-  ArgumentNode
+/** A synthetic node representing the values of variables captured by a comprehension after the output has been computed. */
+class SynthCapturedVariablesArgumentPostUpdateNode extends PostUpdateNodeImpl,
+  TSynthCapturedVariablesArgumentPostUpdateNode
 {
+  ControlFlowNode callable;
+
+  SynthCapturedVariablesArgumentPostUpdateNode() {
+    this = TSynthCapturedVariablesArgumentPostUpdateNode(callable)
+  }
+
+  override string toString() { result = "[post] Capturing closure argument" }
+
+  override Scope getScope() { result = callable.getScope() }
+
+  override Location getLocation() { result = callable.getLocation() }
+
+  override Node getPreUpdateNode() { result = TSynthCapturedVariablesArgumentNode(callable) }
+}
+
+/** A synthetic node representing the values of variables captured by a comprehension. */
+class SynthCompCapturedVariablesArgumentNode extends Node, TSynthCompCapturedVariablesArgumentNode {
   Comp comp;
 
   SynthCompCapturedVariablesArgumentNode() { this = TSynthCompCapturedVariablesArgumentNode(comp) }
@@ -1755,7 +1773,11 @@ class SynthCompCapturedVariablesArgumentNode extends Node, TSynthCompCapturedVar
   override Location getLocation() { result = comp.getLocation() }
 
   Comp getComprehension() { result = comp }
+}
 
+class SynthCompCapturedVariablesArgumentNodeAsArgumentNode extends SynthCompCapturedVariablesArgumentNode,
+  ArgumentNode
+{
   override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
     call.(ComprehensionCall).getComprehension() = comp and
     pos.isLambdaSelf()

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -76,15 +76,7 @@
     node.getNode() = any(Comp c).getIterable()
   } or
   /** A node representing a global (module-level) variable in a specific module. */
-  TModuleVariableNode(Module m, GlobalVariable v) {
-    v.getScope() = m and
-    (
-      v.escapes()
-      or
-      isAccessedThroughImportStar(m) and
-      ImportStar::globalNameDefinedInModule(v.getId(), m)
-    )
-  } or
+  TModuleVariableNode(Module m, GlobalVariable v) { v.getScope() = m } or
   /**
    * A synthetic node representing that an iterable sequence flows to consumer.
    */
@@ -129,6 +121,12 @@
     f = any(VariableCapture::CapturedVariable v).getACapturingScope() and
     exists(TFunction(f))
   } or
+  TSynthCapturedVariablesArgumentNode(ControlFlowNode callable) {
+    callable = any(CallNode c).getFunction()
+  } or
+  TSynthCapturedVariablesArgumentPostUpdateNode(ControlFlowNode callable) {
+    callable = any(CallNode c).getFunction()
+  } or
   /** A synthetic node representing the values of variables captured by a comprehension. */
   TSynthCompCapturedVariablesArgumentNode(Comp comp) {
     comp.getFunction() = any(VariableCapture::CapturedVariable v).getACapturingScope()
@@ -440,13 +438,17 @@
 
   /** Gets a node that reads this variable. */
   Node getARead() {
-    result.asCfgNode() = var.getALoad().getAFlowNode() and
-    // Ignore reads that happen when the module is imported. These are only executed once.
-    not result.getScope() = mod
+    result = this.getALocalRead()
     or
     this = import_star_read(result)
   }
 
+  /** Gets a node that reads this variable, excluding reads that happen through `from ... import *`. */
+  Node getALocalRead() {
+    result.asCfgNode() = var.getALoad().getAFlowNode() and
+    not result.getScope() = mod
+  }
+
   /** Gets an `EssaNode` that corresponds to an assignment of this global variable. */
   Node getAWrite() {
     any(EssaNodeDefinition def).definedBy(var, result.asCfgNode().(DefinitionNode))
@@ -466,8 +468,6 @@
   override Location getLocation() { result = mod.getLocation() }
 }
 
-private predicate isAccessedThroughImportStar(Module m) { m = ImportStar::getStarImported(_) }
-
 private ModuleVariableNode import_star_read(Node n) {
   resolved_import_star_module(result.getModule(), result.getVariable().getId(), n)
 }

python/ql/lib/semmle/python/dataflow/new/internal/LocalSources.qll

@@ -67,7 +67,7 @@ class LocalSourceNode extends Node {
     or
     // We explicitly include any read of a global variable, as some of these may have local flow going
     // into them.
-    this = any(ModuleVariableNode mvn).getARead()
+    this = any(ModuleVariableNode v).getALocalRead()
     or
     // We include all scope entry definitions, as these act as the local source within the scope they
     // enter.
@@ -248,7 +248,7 @@ private module Cached {
   pragma[nomagic]
   private predicate localSourceFlowStep(Node nodeFrom, Node nodeTo) {
     simpleLocalFlowStep(nodeFrom, nodeTo, _) and
-    not nodeTo = any(ModuleVariableNode v).getARead()
+    not nodeTo = any(ModuleVariableNode v).getALocalRead()
   }
 
   /**

python/ql/lib/utils/test/dataflow/MaximalFlowTest.qll

@@ -8,7 +8,9 @@ module MaximalFlowTest implements FlowTestSig {
 
   predicate relevantFlow(DataFlow::Node source, DataFlow::Node sink) {
     source != sink and
-    MaximalFlows::flow(source, sink)
+    MaximalFlows::flow(source, sink) and
+    // exclude ModuleVariableNodes (which have location 0:0:0:0)
+    not sink instanceof DataFlow::ModuleVariableNode
   }
 }
 

python/ql/test/experimental/attrs/AttrWrites.expected

@@ -1,4 +1,5 @@
 | test.py:5:9:5:16 | ControlFlowNode for __init__ | test.py:4:1:4:20 | ControlFlowNode for ClassExpr | __init__ | test.py:5:5:5:28 | ControlFlowNode for FunctionExpr |
 | test.py:6:9:6:16 | ControlFlowNode for Attribute | test.py:6:9:6:12 | ControlFlowNode for self | foo | test.py:6:20:6:22 | ControlFlowNode for foo |
+| test.py:9:1:9:9 | ControlFlowNode for Attribute | test.py:0:0:0:0 | ModuleVariableNode in Module test for myobj | foo | test.py:9:13:9:17 | ControlFlowNode for StringLiteral |
 | test.py:9:1:9:9 | ControlFlowNode for Attribute | test.py:9:1:9:5 | ControlFlowNode for myobj | foo | test.py:9:13:9:17 | ControlFlowNode for StringLiteral |
 | test.py:12:1:12:25 | ControlFlowNode for setattr() | test.py:12:9:12:13 | ControlFlowNode for myobj | foo | test.py:12:23:12:24 | ControlFlowNode for IntegerLiteral |

python/ql/test/library-tests/dataflow/basic/global.expected

@@ -1,33 +1,43 @@
+| test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:7:5:7:17 | ControlFlowNode for obfuscated_id |
+| test.py:1:5:1:17 | ControlFlowNode for obfuscated_id | test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id |
 | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id | test.py:7:5:7:17 | ControlFlowNode for obfuscated_id |
+| test.py:1:19:1:19 | ControlFlowNode for x | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:2:3:2:3 | ControlFlowNode for y |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:2:7:2:7 | ControlFlowNode for x |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:3:3:3:3 | ControlFlowNode for z |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:3:7:3:7 | ControlFlowNode for y |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:4:10:4:10 | ControlFlowNode for z |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
+| test.py:2:3:2:3 | ControlFlowNode for y | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:2:3:2:3 | ControlFlowNode for y | test.py:3:3:3:3 | ControlFlowNode for z |
 | test.py:2:3:2:3 | ControlFlowNode for y | test.py:3:7:3:7 | ControlFlowNode for y |
 | test.py:2:3:2:3 | ControlFlowNode for y | test.py:4:10:4:10 | ControlFlowNode for z |
 | test.py:2:3:2:3 | ControlFlowNode for y | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:2:3:2:3 | ControlFlowNode for y | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
+| test.py:2:7:2:7 | ControlFlowNode for x | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:2:7:2:7 | ControlFlowNode for x | test.py:2:3:2:3 | ControlFlowNode for y |
 | test.py:2:7:2:7 | ControlFlowNode for x | test.py:3:3:3:3 | ControlFlowNode for z |
 | test.py:2:7:2:7 | ControlFlowNode for x | test.py:3:7:3:7 | ControlFlowNode for y |
 | test.py:2:7:2:7 | ControlFlowNode for x | test.py:4:10:4:10 | ControlFlowNode for z |
 | test.py:2:7:2:7 | ControlFlowNode for x | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:2:7:2:7 | ControlFlowNode for x | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
+| test.py:3:3:3:3 | ControlFlowNode for z | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:3:3:3:3 | ControlFlowNode for z | test.py:4:10:4:10 | ControlFlowNode for z |
 | test.py:3:3:3:3 | ControlFlowNode for z | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:3:3:3:3 | ControlFlowNode for z | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
+| test.py:3:7:3:7 | ControlFlowNode for y | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:3:7:3:7 | ControlFlowNode for y | test.py:3:3:3:3 | ControlFlowNode for z |
 | test.py:3:7:3:7 | ControlFlowNode for y | test.py:4:10:4:10 | ControlFlowNode for z |
 | test.py:3:7:3:7 | ControlFlowNode for y | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:3:7:3:7 | ControlFlowNode for y | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
+| test.py:4:10:4:10 | ControlFlowNode for z | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:4:10:4:10 | ControlFlowNode for z | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:4:10:4:10 | ControlFlowNode for z | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
+| test.py:6:1:6:1 | ControlFlowNode for a | test.py:0:0:0:0 | ModuleVariableNode in Module test for a |
+| test.py:6:1:6:1 | ControlFlowNode for a | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:1:19:1:19 | ControlFlowNode for x |
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:2:3:2:3 | ControlFlowNode for y |
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:2:7:2:7 | ControlFlowNode for x |
@@ -37,6 +47,8 @@
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:7:19:7:19 | ControlFlowNode for a |
+| test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:0:0:0:0 | ModuleVariableNode in Module test for a |
+| test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:1:19:1:19 | ControlFlowNode for x |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:2:3:2:3 | ControlFlowNode for y |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:2:7:2:7 | ControlFlowNode for x |
@@ -47,7 +59,10 @@
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:7:19:7:19 | ControlFlowNode for a |
+| test.py:7:1:7:1 | ControlFlowNode for b | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
+| test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() | test.py:7:1:7:1 | ControlFlowNode for b |
+| test.py:7:19:7:19 | ControlFlowNode for a | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:7:19:7:19 | ControlFlowNode for a | test.py:1:19:1:19 | ControlFlowNode for x |
 | test.py:7:19:7:19 | ControlFlowNode for a | test.py:2:3:2:3 | ControlFlowNode for y |
 | test.py:7:19:7:19 | ControlFlowNode for a | test.py:2:7:2:7 | ControlFlowNode for x |

python/ql/test/library-tests/dataflow/basic/globalStep.expected

@@ -1,5 +1,6 @@
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id |
+| test.py:1:5:1:17 | ControlFlowNode for obfuscated_id | test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id |
 | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id | test.py:7:5:7:17 | ControlFlowNode for obfuscated_id |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:2:3:2:3 | ControlFlowNode for y |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:2:3:2:3 | ControlFlowNode for y |
@@ -31,10 +32,13 @@
 | test.py:3:7:3:7 | ControlFlowNode for y | test.py:3:3:3:3 | ControlFlowNode for z |
 | test.py:4:10:4:10 | ControlFlowNode for z | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
 | test.py:4:10:4:10 | ControlFlowNode for z | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() |
+| test.py:6:1:6:1 | ControlFlowNode for a | test.py:0:0:0:0 | ModuleVariableNode in Module test for a |
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:7:19:7:19 | ControlFlowNode for a |
 | test.py:6:1:6:1 | ControlFlowNode for a | test.py:7:19:7:19 | ControlFlowNode for a |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:6:1:6:1 | ControlFlowNode for a |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:6:1:6:1 | ControlFlowNode for a |
+| test.py:7:1:7:1 | ControlFlowNode for b | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
+| test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:7:19:7:19 | ControlFlowNode for a | test.py:1:19:1:19 | ControlFlowNode for x |
 | test.py:7:19:7:19 | ControlFlowNode for a | test.py:1:19:1:19 | ControlFlowNode for x |

python/ql/test/library-tests/dataflow/basic/local.expected

@@ -1,3 +1,8 @@
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for __name__ | test.py:0:0:0:0 | ModuleVariableNode in Module test for __name__ |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for __package__ | test.py:0:0:0:0 | ModuleVariableNode in Module test for __package__ |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for a | test.py:0:0:0:0 | ModuleVariableNode in Module test for a |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for b | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id | test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:7:5:7:17 | ControlFlowNode for obfuscated_id |

python/ql/test/library-tests/dataflow/basic/maximalFlows.expected

@@ -1,7 +1,12 @@
+| test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr | test.py:7:5:7:17 | ControlFlowNode for obfuscated_id |
+| test.py:1:19:1:19 | ControlFlowNode for x | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:4:10:4:10 | ControlFlowNode for z |
 | test.py:1:19:1:19 | ControlFlowNode for x | test.py:7:1:7:1 | ControlFlowNode for b |
+| test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:0:0:0:0 | ModuleVariableNode in Module test for a |
+| test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:4:10:4:10 | ControlFlowNode for z |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:7:1:7:1 | ControlFlowNode for b |
 | test.py:6:5:6:6 | ControlFlowNode for IntegerLiteral | test.py:7:19:7:19 | ControlFlowNode for a |
+| test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() | test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
 | test.py:7:5:7:20 | ControlFlowNode for obfuscated_id() | test.py:7:1:7:1 | ControlFlowNode for b |

python/ql/test/library-tests/dataflow/basic/sinks.expected

@@ -1,3 +1,8 @@
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for __name__ |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for __package__ |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for a |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr |
 | test.py:1:1:1:21 | SynthDictSplatParameterNode |
 | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id |

python/ql/test/library-tests/dataflow/basic/sources.expected

@@ -1,3 +1,8 @@
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for __name__ |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for __package__ |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for a |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for b |
+| test.py:0:0:0:0 | ModuleVariableNode in Module test for obfuscated_id |
 | test.py:1:1:1:21 | ControlFlowNode for FunctionExpr |
 | test.py:1:1:1:21 | SynthDictSplatParameterNode |
 | test.py:1:5:1:17 | ControlFlowNode for obfuscated_id |