diff --git a/Src/IronPython/Compiler/Ast/Comprehension.cs b/Src/IronPython/Compiler/Ast/Comprehension.cs
index 96e547096..19522963b 100644
--- a/Src/IronPython/Compiler/Ast/Comprehension.cs
+++ b/Src/IronPython/Compiler/Ast/Comprehension.cs
@@ -251,7 +251,7 @@ internal override PythonVariable BindReference(PythonNameBinder binder, PythonRe
}
internal override Ast GetVariableExpression(PythonVariable variable) {
- if (variable.IsGlobal) {
+ if (variable.Kind is VariableKind.Global) {
return GlobalParent.ModuleVariables[variable];
}
diff --git a/Src/IronPython/Compiler/Ast/FlowChecker.cs b/Src/IronPython/Compiler/Ast/FlowChecker.cs
index 413a075e3..633075c46 100644
--- a/Src/IronPython/Compiler/Ast/FlowChecker.cs
+++ b/Src/IronPython/Compiler/Ast/FlowChecker.cs
@@ -56,7 +56,7 @@
*
* The bit arrays in the flow checker hold the state and upon encountering NameExpr we figure
* out whether the name has not yet been initialized at all (in which case we need to emit the
- * first explicit assignment to Uninitialized.instance and guard the use with an inlined check
+ * first explicit assignment to Uninitialized.instance and guard the use with an inlined check)
* or whether it is definitely assigned (we don't need to inline the check)
* or whether it may be uninitialized, in which case we must only guard the use by inlining the Uninitialized check
*
@@ -97,7 +97,7 @@ public override bool Walk(IndexExpression node) {
node.Walk(_fc);
return false;
}
-
+
public override bool Walk(ParenthesisExpression node) {
return true;
}
@@ -109,14 +109,13 @@ public override bool Walk(TupleExpression node) {
public override bool Walk(ListExpression node) {
return true;
}
-
+
public override bool Walk(Parameter node) {
_fc.Define(node.Name);
return true;
}
}
- // TODO: probably obsolete, PythonVariable.Deleted does the job better (across scopes)
internal class FlowDeleter : PythonWalkerNonRecursive {
private readonly FlowChecker _fc;
diff --git a/Src/IronPython/Compiler/Ast/PythonAst.cs b/Src/IronPython/Compiler/Ast/PythonAst.cs
index 7f29ce5b7..13edfc015 100644
--- a/Src/IronPython/Compiler/Ast/PythonAst.cs
+++ b/Src/IronPython/Compiler/Ast/PythonAst.cs
@@ -276,13 +276,20 @@ internal ModuleContext ModuleContext {
///
internal PythonVariable/*!*/ EnsureGlobalVariable(string name) {
PythonVariable variable;
- if (!TryGetVariable(name, out variable)) {
+ if (TryGetVariable(name, out variable)) {
+ variable.LiftToGlobal();
+ } else {
variable = CreateVariable(name, VariableKind.Global);
}
return variable;
}
+ internal override MSAst.Expression GetVariableExpression(PythonVariable variable) {
+ Debug.Assert(_globalVariables.ContainsKey(variable));
+ return _globalVariables[variable];
+ }
+
#endregion
#region MSASt.Expression Overrides
diff --git a/Src/IronPython/Compiler/Ast/PythonNameBinder.cs b/Src/IronPython/Compiler/Ast/PythonNameBinder.cs
index 3708a36ac..4534b3dec 100644
--- a/Src/IronPython/Compiler/Ast/PythonNameBinder.cs
+++ b/Src/IronPython/Compiler/Ast/PythonNameBinder.cs
@@ -11,28 +11,35 @@
using Microsoft.Scripting.Runtime;
using Microsoft.Scripting.Utils;
+using IronPython.Runtime;
+
/*
* The name binding:
*
- * The name binding happens in 2 passes.
- * In the first pass (full recursive walk of the AST) we resolve locals.
- * The second pass uses the "processed" list of all context statements (functions and class
- * bodies) and has each context statement resolve its free variables to determine whether
- * they are globals or references to lexically enclosing scopes.
+ * The name binding happens in 4 passes.
+ *
+ * The first pass is a full recursive walk of the AST. During this walk, all scopes are established
+ * (created by functions, classes, comprehensions, generators, and the AST root),
+ * and in each scope all variables are collected that are defined (local or parameter)
+ * or declared (global or nonlocal) there. Also, for each scope, all references used in the
+ * scope are collected but kept unresolved as yet.
+ *
+ * The second pass uses the collected stack of all scopes and has each scope resolve its references.
+ * The references can be resolved locally within the scope or as free variables,
+ * either as globals or as references to lexically enclosing scopes.
*
- * The second pass happens in post-order (the context statement is added into the "processed"
- * list after processing its nested functions/statements). This way, when the function is
- * processing its free variables, it also knows already which of its locals are being lifted
- * to the closure and can report error if such closure variable is being deleted.
+ * The second pass happens in post-order (a scope is processed after processing all its nested scopes).
+ * Consequently, when the scope is processing its free variables, it also knows already
+ * which of its locals are being lifted to the closure.
*
- * This is illegal in Python:
+ * The third pass goes over all the scopes again, this time in pre-order (except for the root AST).
+ * In this pass, all scopes build theirs closures, if needed. Becasue nested scopes are processed
+ * after their encompassing scope, scopes may use the already prepared closure
+ * from their parent.
*
- * def f():
- * x = 10
- * if (cond): del x # illegal because x is a closure variable
- * def g():
- * print x
- * TODO: the example above is no longer valid; also the description of the name binding algorithm does not match the code
+ * During the fourth pass, each scope is being inspected by the FlowChecker,
+ * which analyzes the data flow in the scope. Information collected during this analysis
+ * allows for some optimizations later on, when the expression trees are being constructed.
*/
namespace IronPython.Compiler.Ast {
@@ -58,37 +65,6 @@ public override bool Walk(ListExpression node) {
}
}
- internal class ParameterBinder : PythonWalkerNonRecursive {
- private PythonNameBinder _binder;
- public ParameterBinder(PythonNameBinder binder) {
- _binder = binder;
- }
-
- public override bool Walk(Parameter node) {
- node.Parent = _binder._currentScope;
- node.PythonVariable = _binder.DefineParameter(node.Name);
- return false;
- }
-
- // TODO: obsolete?
- private void WalkTuple(TupleExpression tuple) {
- tuple.Parent = _binder._currentScope;
- foreach (Expression innerNode in tuple.Items) {
- if (innerNode is NameExpression name) {
- _binder.DefineName(name.Name);
- name.Parent = _binder._currentScope;
- name.Reference = _binder.Reference(name.Name);
- } else {
- WalkTuple((TupleExpression)innerNode);
- }
- }
- }
- public override bool Walk(TupleExpression node) {
- node.Parent = _binder._currentScope;
- return true;
- }
- }
-
internal class DeleteBinder : PythonWalkerNonRecursive {
private PythonNameBinder _binder;
public DeleteBinder(PythonNameBinder binder) {
@@ -111,7 +87,6 @@ internal class PythonNameBinder : PythonWalker {
private readonly DefineBinder _define;
private readonly DeleteBinder _delete;
- private readonly ParameterBinder _parameter;
#endregion
@@ -120,7 +95,6 @@ internal class PythonNameBinder : PythonWalker {
private PythonNameBinder(CompilerContext context) {
_define = new DefineBinder(this);
_delete = new DeleteBinder(this);
- _parameter = new ParameterBinder(this);
Context = context;
}
@@ -197,12 +171,11 @@ internal PythonVariable DefineDeleted(string name) {
}
internal void ReportSyntaxWarning(string message, Node node) {
- Context.Errors.Add(Context.SourceUnit, message, node.Span, -1, Severity.Warning);
+ Context.Errors.Add(Context.SourceUnit, message, node.Span, ErrorCodes.SyntaxError, Severity.Warning);
}
internal void ReportSyntaxError(string message, Node node) {
- // TODO: Change the error code (-1)
- Context.Errors.Add(Context.SourceUnit, message, node.Span, -1, Severity.FatalError);
+ Context.Errors.Add(Context.SourceUnit, message, node.Span, ErrorCodes.SyntaxError, Severity.FatalError);
}
#region AstBinder Overrides
@@ -635,12 +608,12 @@ public override bool Walk(FromImportStatement node) {
// FunctionDefinition
public override bool Walk(FunctionDefinition node) {
node._nameVariable = _globalScope.EnsureGlobalVariable("__name__");
-
+
// Name is defined in the enclosing context
if (!node.IsLambda) {
node.PythonVariable = DefineName(node.Name);
}
-
+
// process the default arg values in the outer context
foreach (Parameter p in node.Parameters) {
p.DefaultValue?.Walk(this);
@@ -658,7 +631,8 @@ public override bool Walk(FunctionDefinition node) {
PushScope(node);
foreach (Parameter p in node.Parameters) {
- p.Walk(_parameter);
+ p.Parent = _currentScope;
+ p.PythonVariable = DefineParameter(p.Name);
}
node.Body.Walk(this);
@@ -705,9 +679,8 @@ public override bool Walk(GlobalStatement node) {
ReportSyntaxError($"name '{n}' is used prior to global declaration", node);
}
- // Create the variable in the global context and mark it as global
+ // Create the variable in the global context or mark it as global
PythonVariable variable = _globalScope.EnsureGlobalVariable(n);
- variable.Kind = VariableKind.Global;
if (conflict == null) {
// no previously defined variables, add it to the current scope
diff --git a/Src/IronPython/Compiler/Ast/PythonVariable.cs b/Src/IronPython/Compiler/Ast/PythonVariable.cs
index 7d1b3e889..ccdf7eb35 100644
--- a/Src/IronPython/Compiler/Ast/PythonVariable.cs
+++ b/Src/IronPython/Compiler/Ast/PythonVariable.cs
@@ -21,6 +21,7 @@ public class PythonVariable {
public PythonVariable(string name, VariableKind kind, ScopeStatement/*!*/ scope) {
Assert.NotNull(scope);
+ Debug.Assert(kind != VariableKind.Global || scope.IsGlobal);
Debug.Assert(kind != VariableKind.Nonlocal || !scope.IsGlobal);
Name = name;
Kind = kind;
@@ -32,18 +33,22 @@ public PythonVariable(string name, VariableKind kind, ScopeStatement/*!*/ scope)
///
public string Name { get; }
- public bool IsGlobal {
- get {
- return Kind == VariableKind.Global || Scope.IsGlobal;
- }
- }
-
///
/// The original scope in which the variable is defined.
///
public ScopeStatement Scope { get; }
- public VariableKind Kind { get; set; } // TODO: make readonly
+ public VariableKind Kind { get; private set; }
+
+ ///
+ /// Transform a local kind of variable in a global scope (yes, they can be created)
+ /// into a global kind, hence explicitly accessible to nested local scopes.
+ ///
+ internal void LiftToGlobal() {
+ Debug.Assert(Scope.IsGlobal);
+ Debug.Assert(Kind is not VariableKind.Parameter and not VariableKind.Nonlocal);
+ Kind = VariableKind.Global;
+ }
///
/// The actual variable represented by this variable instance.
diff --git a/Src/IronPython/Compiler/Ast/ScopeStatement.cs b/Src/IronPython/Compiler/Ast/ScopeStatement.cs
index 7a857016a..a90c150e0 100644
--- a/Src/IronPython/Compiler/Ast/ScopeStatement.cs
+++ b/Src/IronPython/Compiler/Ast/ScopeStatement.cs
@@ -384,8 +384,7 @@ internal virtual void FinishBind(PythonNameBinder binder) {
if (Variables != null) {
foreach (PythonVariable variable in Variables.Values) {
if (!HasClosureVariable(closureVariables, variable) &&
- !variable.IsGlobal &&
- variable.Kind != VariableKind.Nonlocal &&
+ variable.Kind is not VariableKind.Global and not VariableKind.Nonlocal &&
(variable.AccessedInNestedScope || ExposesLocalVariable(variable))) {
if (closureVariables == null) {
@@ -700,7 +699,7 @@ private MSAst.Expression GetClosureCell(ClosureInfo variable) {
internal virtual MSAst.Expression GetVariableExpression(PythonVariable variable) {
Assert.NotNull(variable?.LimitVariable);
- if (variable.IsGlobal) {
+ if (variable.Kind is VariableKind.Global) {
return GlobalParent.ModuleVariables[variable];
}
diff --git a/Src/IronPython/Compiler/Ast/SerializedScopeStatement.cs b/Src/IronPython/Compiler/Ast/SerializedScopeStatement.cs
index c3079e01b..35e3734a1 100644
--- a/Src/IronPython/Compiler/Ast/SerializedScopeStatement.cs
+++ b/Src/IronPython/Compiler/Ast/SerializedScopeStatement.cs
@@ -15,6 +15,7 @@
namespace IronPython.Compiler.Ast {
///
/// Fake ScopeStatement for FunctionCode's to hold on to after we have deserialized pre-compiled code.
+ /// Will never be a subject of name binding.
///
internal class SerializedScopeStatement : ScopeStatement {
private readonly string _name;
@@ -74,29 +75,15 @@ public override void Walk(PythonWalker walker) {
throw new InvalidOperationException();
}
- public override string Name {
- get {
- return _name;
- }
- }
+ public override string Name => _name;
- internal override string Filename {
- get {
- return _filename;
- }
- }
+ internal override string Filename => _filename;
- internal override FunctionAttributes Flags {
- get {
- return _flags;
- }
- }
+ internal override FunctionAttributes Flags => _flags;
- internal override string[] ParameterNames {
- get {
- return _parameterNames;
- }
- }
+ internal override bool IsGlobal => true;
+
+ internal override string[] ParameterNames => _parameterNames;
internal override int ArgCount => _argCount;
diff --git a/Tests/hosting/editor_svcs/test_errorlistener.py b/Tests/hosting/editor_svcs/test_errorlistener.py
index 923f787df..db26946a6 100644
--- a/Tests/hosting/editor_svcs/test_errorlistener.py
+++ b/Tests/hosting/editor_svcs/test_errorlistener.py
@@ -137,7 +137,7 @@ def test_multiple_erroneous_statements(self):
def test_warning(self):
expected = [
- ("name 'a' is assigned to before global declaration", "global a", -1, self.FatalError), # reports as error since 3.6
+ ("name 'a' is assigned to before global declaration", "global a", 0x10, self.FatalError), # reports as error since 3.6
]
code = """\
def foo():
@@ -148,8 +148,8 @@ def foo():
def test_should_report_multiple_warnings_negative(self):
"Bug #17541, http://www.codeplex.com/IronPython/WorkItem/View.aspx?WorkItemId=17541"
expected = [
- ("Variable a assigned before global declaration", "global a", -1, self.Warning),
- ("Variable b assigned before global declaration", "global b", -1, self.Warning),
+ ("Variable a assigned before global declaration", "global a", 0x10, self.Warning),
+ ("Variable b assigned before global declaration", "global b", 0x10, self.Warning),
]
code = """\
def foo():
@@ -164,8 +164,8 @@ def bar():
def test_should_report_both_errors_and_warnings_negative(self):
"Bug #17541, http://www.codeplex.com/IronPython/WorkItem/View.aspx?WorkItemId=17541"
expected = [
- ("can't assign to keyword", "None", -1, self.Error),
- ("Variable a assigned before global declaration", "global a", -1, self.Warning),
+ ("can't assign to keyword", "None", 0x10, self.Error),
+ ("Variable a assigned before global declaration", "global a", 0x10, self.Warning),
]
code = """\
None = 2
diff --git a/WhatsNewInPython30.md b/WhatsNewInPython30.md
index 4d9ff67c5..f652d1450 100644
--- a/WhatsNewInPython30.md
+++ b/WhatsNewInPython30.md
@@ -46,7 +46,7 @@ New Syntax
- [ ] [PEP 3107][]: Function argument and return value annotations. This provides a standardized way of annotating a function's parameters and return value. There are no semantics attached to such annotations except that they can be introspected at runtime using the `__annotations__` attribute. The intent is to encourage experimentation through metaclasses, decorators or frameworks.
- [ ] [PEP 3102][]: Keyword-only arguments. Named parameters occurring after `*args` in the parameter list must be specified using keyword syntax in the call. You can also use a bare `*` in the parameter list to indicate that you don't accept a variable-length argument list, but you do have keyword-only arguments.
- [ ] Keyword arguments are allowed after the list of base classes in a class definition. This is used by the new convention for specifying a metaclass (see next section), but can be used for other purposes as well, as long as the metaclass supports it.
-- [ ] [PEP 3104][]: `nonlocal` statement. Using `nonlocal x` you can now assign directly to a variable in an outer (but non-`global`) scope. `nonlocal` is a new reserved word.
+- [x] [PEP 3104][]: `nonlocal` statement. Using `nonlocal x` you can now assign directly to a variable in an outer (but non-`global`) scope. `nonlocal` is a new reserved word.
- [ ] [PEP 3132][]: Extended Iterable Unpacking. You can now write things like `a, b, *rest = some_sequence`. And even `*rest, a = stuff`. The `rest` object is always a (possibly empty) list; the right-hand side may be any iterable
- [x] Dictionary comprehensions: `{k: v for k, v in stuff}` means the same thing as `dict(stuff)` but is more flexible. (This is [PEP 0274][] vindicated. :-)
- [x] Set literals, e.g. `{1, 2}`. Note that `{}` is an empty dictionary; use `set()` for an empty set. Set comprehensions are also supported; e.g., `{x for x in stuff}` means the same thing as `set(stuff)` but is more flexible.