Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Move the compressor to Rhino 1.7R2. This fixes the build.xml to point…

… to the new

jar, adds the new jar under the lib/, removes the old jar, makes YUI's
JavaScriptCompressor.java aware of "let" and "yield" by setting the env to 1.7 by
default, and adds the Rhino 1.7 changes to the src/org/mozilla/javascript tree.
commit 54f3bf2cb7d64e2ba22dca2c52917aa0cbc9095c 1 parent ed236d6
@tml tml authored
View
8 build.xml
@@ -20,10 +20,10 @@
includes="**/*.java"
deprecation="off"
debug="on"
- source="1.4">
+ source="1.6">
<classpath>
<pathelement location="${lib.dir}/jargs-1.0.jar"/>
- <pathelement location="${lib.dir}/rhino-1.6R7.jar"/>
+ <pathelement location="${lib.dir}/rhino-1.7R2.jar"/>
</classpath>
</javac>
</target>
@@ -33,7 +33,7 @@
<!-- The order is important here. Rhino MUST be unjarred first!
(some of our own classes will override the Rhino classes) -->
<unjar src="${lib.dir}/jargs-1.0.jar" dest="${build.dir}/jar"/>
- <unjar src="${lib.dir}/rhino-1.6R7.jar" dest="${build.dir}/jar"/>
+ <unjar src="${lib.dir}/rhino-1.7R2.jar" dest="${build.dir}/jar"/>
<copy todir="${build.dir}/jar">
<fileset dir="${build.dir}/classes" includes="**/*.class"/>
</copy>
@@ -63,4 +63,4 @@
includes="${dist.package.name}/**/*"/>
</target>
-</project>
+</project>
View
BIN  lib/rhino-1.6R7.jar
Binary file not shown
View
BIN  lib/rhino-1.7R2.jar
Binary file not shown
View
3  src/com/yahoo/platform/yui/compressor/JavaScriptCompressor.java
@@ -170,6 +170,8 @@
literals.put(new Integer(Token.DOTDOT), "..");
literals.put(new Integer(Token.DOTQUERY), ".(");
literals.put(new Integer(Token.XMLATTR), "@");
+ literals.put(new Integer(Token.LET), "let ");
+ literals.put(new Integer(Token.YIELD), "yield ");
// See http://developer.mozilla.org/en/docs/Core_JavaScript_1.5_Reference:Reserved_Words
@@ -308,6 +310,7 @@ private static ArrayList parse(Reader in, ErrorReporter reporter)
throws IOException, EvaluatorException {
CompilerEnvirons env = new CompilerEnvirons();
+ env.setLanguageVersion(Context.VERSION_1_7);
Parser parser = new Parser(env, reporter);
parser.parse(in, null, 1);
String source = parser.getEncodedSource();
View
10 src/org/mozilla/javascript/Decompiler.java
@@ -588,6 +588,10 @@ else if (nextToken == Token.NAME) {
result.append("var ");
break;
+ case Token.LET:
+ result.append("let ");
+ break;
+
case Token.SEMI:
result.append(';');
if (Token.EOL != getNext(source, length, i)) {
@@ -745,7 +749,11 @@ else if (nextToken == Token.NAME) {
case Token.CONST:
result.append("const ");
break;
-
+
+ case Token.YIELD:
+ result.append("yield ");
+ break;
+
case Token.NOT:
result.append('!');
break;
View
722 src/org/mozilla/javascript/Parser.java
@@ -1,35 +1,36 @@
/* ***** BEGIN LICENSE BLOCK *****
- *
- * Version: MPL 1.1
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
- * See the License for the specific language governing rights and
- * limitations under the License.
- *
- * The Original Code is org/mozilla/javascript/Parser.java,
- * a component of the Rhino Library ( http://www.mozilla.org/rhino/ )
- * This file is a modification of the Original Code developed
- * for YUI Compressor.
- *
- * The Initial Developer of the Original Code is Mozilla Foundation
- *
- * Copyright (c) 2009 Mozilla Foundation. All Rights Reserved.
- *
- * Contributor(s): Yahoo! Inc. 2009
- *
- * ***** END LICENSE BLOCK ***** */
+*
+* Version: MPL 1.1
+*
+* The contents of this file are subject to the Mozilla Public License
+* Version 1.1 (the "License"); you may not use this file except in
+* compliance with the License. You may obtain a copy of the License
+* at http://www.mozilla.org/MPL/
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
+* See the License for the specific language governing rights and
+* limitations under the License.
+*
+* The Original Code is org/mozilla/javascript/Parser.java,
+* a component of the Rhino Library ( http://www.mozilla.org/rhino/ )
+* This file is a modification of the Original Code developed
+* for YUI Compressor.
+*
+* The Initial Developer of the Original Code is Mozilla Foundation
+*
+* Copyright (c) 2009 Mozilla Foundation. All Rights Reserved.
+*
+* Contributor(s): Yahoo! Inc. 2009
+*
+* ***** END LICENSE BLOCK ***** */
package org.mozilla.javascript;
import java.io.Reader;
import java.io.IOException;
-import java.util.Hashtable;
+import java.util.HashMap;
+import java.util.Map;
/**
* This class implements the JavaScript parser.
@@ -72,13 +73,17 @@
// during function parsing.
// XXX Move to separated class?
ScriptOrFnNode currentScriptOrFn;
+ Node.Scope currentScope;
private int nestingOfWith;
- private Hashtable labelSet; // map of label names into nodes
+ private Map<String,Node> labelSet; // map of label names into nodes
private ObjArray loopSet;
private ObjArray loopAndSwitchSet;
- private boolean hasReturnValue;
- private int functionEndFlags;
+ private int endFlags;
// end of per function variables
+
+ public int getCurrentLineNumber() {
+ return ts.getLineno();
+ }
// Exception to unwind
private static class ParserException extends RuntimeException
@@ -145,30 +150,11 @@ private int peekToken()
{
int tt = currentFlaggedToken;
if (tt == Token.EOF) {
-
- while ((tt = ts.getToken()) == Token.CONDCOMMENT || tt == Token.KEEPCOMMENT) {
- if (tt == Token.CONDCOMMENT) {
- /* Support for JScript conditional comments */
- decompiler.addJScriptConditionalComment(ts.getString());
- } else {
- /* Support for preserved comments */
- decompiler.addPreservedComment(ts.getString());
- }
- }
-
@fruju
fruju added a note

This change appears to have broken support for Preserved Comments.

For example, /*! comment */ now generates a syntax error

@tml Collaborator
tml added a note

Thanks for pointing this out - this was not a feature I was aware of, so I didn't bother preserving the support for it when I brought the new Rhino in. You can find a fix in my fork at tml@1aade20 - since there are 20+ outstanding PR's against this project (including one from me), I haven't bothered to issue a PR on this yet.

@fruju
fruju added a note

Thanks for the quick response. I'd suggest adding a test case for this (I see the project is sorely lacking in Javascript minification test cases)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ tt = ts.getToken();
if (tt == Token.EOL) {
do {
tt = ts.getToken();
-
- if (tt == Token.CONDCOMMENT) {
- /* Support for JScript conditional comments */
- decompiler.addJScriptConditionalComment(ts.getString());
- } else if (tt == Token.KEEPCOMMENT) {
- /* Support for preserved comments */
- decompiler.addPreservedComment(ts.getString());
- }
-
- } while (tt == Token.EOL || tt == Token.CONDCOMMENT || tt == Token.KEEPCOMMENT);
+ } while (tt == Token.EOL);
tt |= TI_AFTER_EOL;
}
currentFlaggedToken = tt;
@@ -263,8 +249,19 @@ boolean insideFunction()
{
return nestingOfFunction != 0;
}
+
+ void pushScope(Node node) {
+ Node.Scope scopeNode = (Node.Scope) node;
+ if (scopeNode.getParentScope() != null) throw Kit.codeBug();
+ scopeNode.setParent(currentScope);
+ currentScope = scopeNode;
+ }
+
+ void popScope() {
+ currentScope = currentScope.getParentScope();
+ }
- private Node enterLoop(Node loopLabel)
+ private Node enterLoop(Node loopLabel, boolean doPushScope)
{
Node loop = nf.createLoopNode(loopLabel, ts.getLineno());
if (loopSet == null) {
@@ -275,13 +272,19 @@ private Node enterLoop(Node loopLabel)
}
loopSet.push(loop);
loopAndSwitchSet.push(loop);
+ if (doPushScope) {
+ pushScope(loop);
+ }
return loop;
}
- private void exitLoop()
+ private void exitLoop(boolean doPopScope)
{
loopSet.pop();
loopAndSwitchSet.pop();
+ if (doPopScope) {
+ popScope();
+ }
}
private Node enterSwitch(Node switchSelector, int lineno)
@@ -343,6 +346,7 @@ private ScriptOrFnNode parse()
this.decompiler = createDecompiler(compilerEnv);
this.nf = new IRFactory(this);
currentScriptOrFn = nf.createScript();
+ currentScope = currentScriptOrFn;
int sourceStartOffset = decompiler.getCurrentOffset();
this.encodedSource = null;
decompiler.addToken(Token.SCRIPT);
@@ -489,6 +493,13 @@ private Node function(int functionType)
if (memberExprNode != null) {
syntheticType = FunctionNode.FUNCTION_EXPRESSION;
+ }
+
+ if (syntheticType != FunctionNode.FUNCTION_EXPRESSION &&
+ name.length() > 0)
+ {
+ // Function statements define a symbol in the enclosing scope
+ defineSymbol(Token.FUNCTION, false, name);
}
boolean nested = insideFunction();
@@ -502,24 +513,26 @@ private Node function(int functionType)
// of with object.
fnNode.itsIgnoreDynamicScope = true;
}
-
int functionIndex = currentScriptOrFn.addFunction(fnNode);
int functionSourceEnd;
ScriptOrFnNode savedScriptOrFn = currentScriptOrFn;
currentScriptOrFn = fnNode;
+ Node.Scope savedCurrentScope = currentScope;
+ currentScope = fnNode;
int savedNestingOfWith = nestingOfWith;
nestingOfWith = 0;
- Hashtable savedLabelSet = labelSet;
+ Map<String,Node> savedLabelSet = labelSet;
labelSet = null;
ObjArray savedLoopSet = loopSet;
loopSet = null;
ObjArray savedLoopAndSwitchSet = loopAndSwitchSet;
loopAndSwitchSet = null;
- boolean savedHasReturnValue = hasReturnValue;
- int savedFunctionEndFlags = functionEndFlags;
+ int savedFunctionEndFlags = endFlags;
+ endFlags = 0;
+ Node destructuring = null;
Node body;
try {
decompiler.addToken(Token.LP);
@@ -529,13 +542,26 @@ private Node function(int functionType)
if (!first)
decompiler.addToken(Token.COMMA);
first = false;
- mustMatchToken(Token.NAME, "msg.no.parm");
- String s = ts.getString();
- if (fnNode.hasParamOrVar(s)) {
- addWarning("msg.dup.parms", s);
+ int tt = peekToken();
+ if (tt == Token.LB || tt == Token.LC) {
+ // Destructuring assignment for parameters: add a
+ // dummy parameter name, and add a statement to the
+ // body to initialize variables from the destructuring
+ // assignment
+ if (destructuring == null) {
+ destructuring = new Node(Token.COMMA);
+ }
+ String parmName = currentScriptOrFn.getNextTempName();
+ defineSymbol(Token.LP, false, parmName);
+ destructuring.addChildToBack(
+ nf.createDestructuringAssignment(Token.VAR,
+ primaryExpr(), nf.createName(parmName)));
+ } else {
+ mustMatchToken(Token.NAME, "msg.no.parm");
+ String s = ts.getString();
+ defineSymbol(Token.LP, false, s);
+ decompiler.addName(s);
}
- fnNode.addParam(s);
- decompiler.addName(s);
} while (matchToken(Token.COMMA));
mustMatchToken(Token.RP, "msg.no.paren.after.parms");
@@ -545,6 +571,10 @@ private Node function(int functionType)
mustMatchToken(Token.LC, "msg.no.brace.body");
decompiler.addEOL(Token.LC);
body = parseFunctionBody();
+ if (destructuring != null) {
+ body.addChildToFront(
+ new Node(Token.EXPR_VOID, destructuring, ts.getLineno()));
+ }
mustMatchToken(Token.RC, "msg.no.brace.after.body");
if (compilerEnv.isStrictMode() && !body.hasConsistentReturnUsage())
@@ -553,7 +583,15 @@ private Node function(int functionType)
: "msg.anon.no.return.value";
addStrictWarning(msg, name);
}
-
+
+ if (syntheticType == FunctionNode.FUNCTION_EXPRESSION &&
+ name.length() > 0 && currentScope.getSymbol(name) == null)
+ {
+ // Function expressions define a name only in the body of the
+ // function, and only if not hidden by a parameter name
+ defineSymbol(Token.FUNCTION, false, name);
+ }
+
decompiler.addToken(Token.RC);
functionSourceEnd = decompiler.markFunctionEnd(functionSourceStart);
if (functionType != FunctionNode.FUNCTION_EXPRESSION) {
@@ -563,13 +601,13 @@ private Node function(int functionType)
}
}
finally {
- hasReturnValue = savedHasReturnValue;
- functionEndFlags = savedFunctionEndFlags;
+ endFlags = savedFunctionEndFlags;
loopAndSwitchSet = savedLoopAndSwitchSet;
loopSet = savedLoopSet;
labelSet = savedLabelSet;
nestingOfWith = savedNestingOfWith;
currentScriptOrFn = savedScriptOrFn;
+ currentScope = savedCurrentScope;
}
fnNode.setEncodedSourceBounds(functionSourceStart, functionSourceEnd);
@@ -577,12 +615,6 @@ private Node function(int functionType)
fnNode.setBaseLineno(baseLineno);
fnNode.setEndLineno(ts.getLineno());
- if (name != null) {
- int index = currentScriptOrFn.getParamOrVarIndex(name);
- if (index >= 0 && index < currentScriptOrFn.getParamCount())
- addStrictWarning("msg.var.hides.arg", name);
- }
-
Node pn = nf.initFunction(fnNode, functionIndex, body, syntheticType);
if (memberExprNode != null) {
pn = nf.createAssignment(Token.ASSIGN, memberExprNode, pn);
@@ -594,13 +626,13 @@ private Node function(int functionType)
return pn;
}
- private Node statements()
+ private Node statements(Node scope)
throws IOException
{
- Node pn = nf.createBlock(ts.getLineno());
+ Node pn = scope != null ? scope : nf.createBlock(ts.getLineno());
int tt;
- while((tt = peekToken()) > Token.EOF && tt != Token.RC) {
+ while ((tt = peekToken()) > Token.EOF && tt != Token.RC) {
nf.addChildToBack(pn, statement());
}
@@ -639,7 +671,7 @@ private Node matchJumpLabelName()
String name = ts.getString();
decompiler.addName(name);
if (labelSet != null) {
- label = (Node)labelSet.get(name);
+ label = labelSet.get(name);
}
if (label == null) {
reportError("msg.undef.label");
@@ -677,21 +709,13 @@ private Node statement()
return nf.createExprStatement(nf.createName("error"), lineno);
}
- /**
- * Whether the "catch (e: e instanceof Exception) { ... }" syntax
- * is implemented.
- */
-
private Node statementHelper(Node statementLabel)
throws IOException, ParserException
{
Node pn = null;
+ int tt = peekToken();
- int tt;
-
- tt = peekToken();
-
- switch(tt) {
+ switch (tt) {
case Token.IF: {
consumeToken();
@@ -766,7 +790,7 @@ private Node statementHelper(Node statementLabel)
nf.addChildToBack(block, statement());
}
- // caseExpression == null => add default lable
+ // caseExpression == null => add default label
nf.addSwitchCase(pn, caseExpression, block);
}
decompiler.addEOL(Token.RC);
@@ -781,7 +805,7 @@ private Node statementHelper(Node statementLabel)
consumeToken();
decompiler.addToken(Token.WHILE);
- Node loop = enterLoop(statementLabel);
+ Node loop = enterLoop(statementLabel, true);
try {
Node cond = condition();
decompiler.addEOL(Token.LC);
@@ -789,7 +813,7 @@ private Node statementHelper(Node statementLabel)
decompiler.addEOL(Token.RC);
pn = nf.createWhile(loop, cond, body);
} finally {
- exitLoop();
+ exitLoop(true);
}
return pn;
}
@@ -799,7 +823,7 @@ private Node statementHelper(Node statementLabel)
decompiler.addToken(Token.DO);
decompiler.addEOL(Token.LC);
- Node loop = enterLoop(statementLabel);
+ Node loop = enterLoop(statementLabel, true);
try {
Node body = statement();
decompiler.addToken(Token.RC);
@@ -808,10 +832,10 @@ private Node statementHelper(Node statementLabel)
Node cond = condition();
pn = nf.createDoWhile(loop, body, cond);
} finally {
- exitLoop();
+ exitLoop(true);
}
- // Always auto-insert semicon to follow SpiderMonkey:
- // It is required by EMAScript but is ignored by the rest of
+ // Always auto-insert semicolon to follow SpiderMonkey:
+ // It is required by ECMAScript but is ignored by the rest of
// world, see bug 238945
matchToken(Token.SEMI);
decompiler.addEOL(Token.SEMI);
@@ -823,13 +847,13 @@ private Node statementHelper(Node statementLabel)
boolean isForEach = false;
decompiler.addToken(Token.FOR);
- Node loop = enterLoop(statementLabel);
+ Node loop = enterLoop(statementLabel, true);
try {
-
- Node init; // Node init is also foo in 'foo in Object'
- Node cond; // Node cond is also object in 'foo in Object'
- Node incr = null; // to kill warning
+ Node init; // Node init is also foo in 'foo in object'
+ Node cond; // Node cond is also object in 'foo in object'
+ Node incr = null;
Node body;
+ int declType = -1;
// See if this is a for each () instead of just a for ()
if (matchToken(Token.NAME)) {
@@ -847,10 +871,12 @@ private Node statementHelper(Node statementLabel)
if (tt == Token.SEMI) {
init = nf.createLeaf(Token.EMPTY);
} else {
- if (tt == Token.VAR) {
+ if (tt == Token.VAR || tt == Token.LET) {
// set init to a var list or initial
- consumeToken(); // consume the 'var' token
- init = variables(Token.FOR);
+ consumeToken(); // consume the token
+ decompiler.addToken(tt);
+ init = variables(true, tt);
+ declType = tt;
}
else {
init = expr(true);
@@ -889,12 +915,13 @@ private Node statementHelper(Node statementLabel)
if (incr == null) {
// cond could be null if 'in obj' got eaten
// by the init node.
- pn = nf.createForIn(loop, init, cond, body, isForEach);
+ pn = nf.createForIn(declType, loop, init, cond, body,
+ isForEach);
} else {
pn = nf.createFor(loop, init, cond, incr, body);
}
} finally {
- exitLoop();
+ exitLoop(true);
}
return pn;
}
@@ -908,6 +935,9 @@ private Node statementHelper(Node statementLabel)
Node finallyblock = null;
decompiler.addToken(Token.TRY);
+ if (peekToken() != Token.LC) {
+ reportError("msg.no.brace.try");
+ }
decompiler.addEOL(Token.LC);
tryblock = statement();
decompiler.addEOL(Token.RC);
@@ -944,7 +974,7 @@ private Node statementHelper(Node statementLabel)
nf.addChildToBack(catchblocks,
nf.createCatch(varName, catchCond,
- statements(),
+ statements(null),
ts.getLineno()));
mustMatchToken(Token.RC, "msg.no.brace.after.body");
@@ -1055,62 +1085,53 @@ private Node statementHelper(Node statementLabel)
case Token.CONST:
case Token.VAR: {
consumeToken();
- pn = variables(tt);
+ decompiler.addToken(tt);
+ pn = variables(false, tt);
break;
}
-
- case Token.RETURN: {
- if (!insideFunction()) {
- reportError("msg.bad.return");
- }
+
+ case Token.LET: {
consumeToken();
- decompiler.addToken(Token.RETURN);
- int lineno = ts.getLineno();
-
- Node retExpr;
- /* This is ugly, but we don't want to require a semicolon. */
- tt = peekTokenOrEOL();
- switch (tt) {
- case Token.SEMI:
- case Token.RC:
- case Token.EOF:
- case Token.EOL:
- case Token.ERROR:
- retExpr = null;
- break;
- default:
- retExpr = expr(false);
- hasReturnValue = true;
- }
- pn = nf.createReturn(retExpr, lineno);
-
- // see if we need a strict mode warning
- if (retExpr == null) {
- if (functionEndFlags == Node.END_RETURNS_VALUE)
- addStrictWarning("msg.return.inconsistent", "");
-
- functionEndFlags |= Node.END_RETURNS;
+ decompiler.addToken(Token.LET);
+ if (peekToken() == Token.LP) {
+ return let(true);
} else {
- if (functionEndFlags == Node.END_RETURNS)
- addStrictWarning("msg.return.inconsistent", "");
-
- functionEndFlags |= Node.END_RETURNS_VALUE;
+ pn = variables(false, tt);
+ if (peekToken() == Token.SEMI)
+ break;
+ return pn;
}
+ }
+ case Token.RETURN:
+ case Token.YIELD: {
+ pn = returnOrYield(tt, false);
break;
}
+ case Token.DEBUGGER:
+ consumeToken();
+ decompiler.addToken(Token.DEBUGGER);
+ pn = nf.createDebugger(ts.getLineno());
+ break;
+
case Token.LC:
consumeToken();
if (statementLabel != null) {
decompiler.addToken(Token.LC);
}
- pn = statements();
- mustMatchToken(Token.RC, "msg.no.brace.block");
- if (statementLabel != null) {
- decompiler.addEOL(Token.RC);
+ Node scope = nf.createScopeNode(Token.BLOCK, ts.getLineno());
+ pushScope(scope);
+ try {
+ statements(scope);
+ mustMatchToken(Token.RC, "msg.no.brace.block");
+ if (statementLabel != null) {
+ decompiler.addEOL(Token.RC);
+ }
+ return scope;
+ } finally {
+ popScope();
}
- return pn;
case Token.ERROR:
// Fall thru, to have a node for error recovery to work on
@@ -1173,7 +1194,7 @@ private Node statementHelper(Node statementLabel)
decompiler.addEOL(Token.COLON);
if (labelSet == null) {
- labelSet = new Hashtable();
+ labelSet = new HashMap<String,Node>();
} else if (labelSet.containsKey(name)) {
reportError("msg.dup.label");
}
@@ -1233,74 +1254,248 @@ private Node statementHelper(Node statementLabel)
}
/**
+ * Returns whether or not the bits in the mask have changed to all set.
+ * @param before bits before change
+ * @param after bits after change
+ * @param mask mask for bits
+ * @return true if all the bits in the mask are set in "after" but not
+ * "before"
+ */
+ private static final boolean nowAllSet(int before, int after, int mask)
+ {
+ return ((before & mask) != mask) && ((after & mask) == mask);
+ }
+
+ private Node returnOrYield(int tt, boolean exprContext)
+ throws IOException, ParserException
+ {
+ if (!insideFunction()) {
+ reportError(tt == Token.RETURN ? "msg.bad.return"
+ : "msg.bad.yield");
+ }
+ consumeToken();
+ decompiler.addToken(tt);
+ int lineno = ts.getLineno();
+
+ Node e;
+ /* This is ugly, but we don't want to require a semicolon. */
+ switch (peekTokenOrEOL()) {
+ case Token.SEMI:
+ case Token.RC:
+ case Token.EOF:
+ case Token.EOL:
+ case Token.ERROR:
+ case Token.RB:
+ case Token.RP:
+ case Token.YIELD:
+ e = null;
+ break;
+ default:
+ e = expr(false);
+ break;
+ }
+
+ int before = endFlags;
+ Node ret;
+
+ if (tt == Token.RETURN) {
+ if (e == null ) {
+ endFlags |= Node.END_RETURNS;
+ } else {
+ endFlags |= Node.END_RETURNS_VALUE;
+ }
+ ret = nf.createReturn(e, lineno);
+
+ // see if we need a strict mode warning
+ if (nowAllSet(before, endFlags,
+ Node.END_RETURNS|Node.END_RETURNS_VALUE))
+ {
+ addStrictWarning("msg.return.inconsistent", "");
+ }
+ } else {
+ endFlags |= Node.END_YIELDS;
+ ret = nf.createYield(e, lineno);
+ if (!exprContext)
+ ret = new Node(Token.EXPR_VOID, ret, lineno);
+ }
+
+ // see if we are mixing yields and value returns.
+ if (nowAllSet(before, endFlags,
+ Node.END_YIELDS|Node.END_RETURNS_VALUE))
+ {
+ String name = ((FunctionNode)currentScriptOrFn).getFunctionName();
+ if (name.length() == 0)
+ addError("msg.anon.generator.returns", "");
+ else
+ addError("msg.generator.returns", name);
+ }
+
+ return ret;
+ }
+
+ /**
* Parse a 'var' or 'const' statement, or a 'var' init list in a for
* statement.
- * @param context A token value: either VAR, CONST or FOR depending on
+ * @param inFor true if we are currently in the midst of the init
+ * clause of a for.
+ * @param declType A token value: either VAR, CONST, or LET depending on
* context.
* @return The parsed statement
* @throws IOException
* @throws ParserException
*/
- private Node variables(int context)
+ private Node variables(boolean inFor, int declType)
throws IOException, ParserException
{
- Node pn;
+ Node result = nf.createVariables(declType, ts.getLineno());
boolean first = true;
-
- if (context == Token.CONST){
- pn = nf.createVariables(Token.CONST, ts.getLineno());
- decompiler.addToken(Token.CONST);
- } else {
- pn = nf.createVariables(Token.VAR, ts.getLineno());
- decompiler.addToken(Token.VAR);
- }
-
for (;;) {
- Node name;
- Node init;
- mustMatchToken(Token.NAME, "msg.bad.var");
- String s = ts.getString();
-
- if (!first)
- decompiler.addToken(Token.COMMA);
- first = false;
-
- decompiler.addName(s);
-
- if (context == Token.CONST) {
- if (!currentScriptOrFn.addConst(s)) {
- // We know it's already defined, since addConst passes if
- // it's not defined at all. The addVar call just confirms
- // what it is.
- if (currentScriptOrFn.addVar(s) != ScriptOrFnNode.DUPLICATE_CONST)
- addError("msg.var.redecl", s);
- else
- addError("msg.const.redecl", s);
- }
+ Node destructuring = null;
+ String s = null;
+ int tt = peekToken();
+ if (tt == Token.LB || tt == Token.LC) {
+ // Destructuring assignment, e.g., var [a,b] = ...
+ destructuring = primaryExpr();
} else {
- int dupState = currentScriptOrFn.addVar(s);
- if (dupState == ScriptOrFnNode.DUPLICATE_CONST)
- addError("msg.const.redecl", s);
- else if (dupState == ScriptOrFnNode.DUPLICATE_PARAMETER)
- addStrictWarning("msg.var.hides.arg", s);
- else if (dupState == ScriptOrFnNode.DUPLICATE_VAR)
- addStrictWarning("msg.var.redecl", s);
+ // Simple variable name
+ mustMatchToken(Token.NAME, "msg.bad.var");
+ s = ts.getString();
+
+ if (!first)
+ decompiler.addToken(Token.COMMA);
+ first = false;
+
+ decompiler.addName(s);
+ defineSymbol(declType, inFor, s);
}
- name = nf.createName(s);
-
- // omitted check for argument hiding
-
+
+ Node init = null;
if (matchToken(Token.ASSIGN)) {
decompiler.addToken(Token.ASSIGN);
-
- init = assignExpr(context == Token.FOR);
- nf.addChildToBack(name, init);
+ init = assignExpr(inFor);
+ }
+
+ if (destructuring != null) {
+ if (init == null) {
+ if (!inFor)
+ reportError("msg.destruct.assign.no.init");
+ nf.addChildToBack(result, destructuring);
+ } else {
+ nf.addChildToBack(result,
+ nf.createDestructuringAssignment(declType,
+ destructuring, init));
+ }
+ } else {
+ Node name = nf.createName(s);
+ if (init != null)
+ nf.addChildToBack(name, init);
+ nf.addChildToBack(result, name);
}
- nf.addChildToBack(pn, name);
+
if (!matchToken(Token.COMMA))
break;
}
- return pn;
+ return result;
+ }
+
+
+ private Node let(boolean isStatement)
+ throws IOException, ParserException
+ {
+ mustMatchToken(Token.LP, "msg.no.paren.after.let");
+ decompiler.addToken(Token.LP);
+ Node result = nf.createScopeNode(Token.LET, ts.getLineno());
+ pushScope(result);
+ try {
+ Node vars = variables(false, Token.LET);
+ nf.addChildToBack(result, vars);
+ mustMatchToken(Token.RP, "msg.no.paren.let");
+ decompiler.addToken(Token.RP);
+ if (isStatement && peekToken() == Token.LC) {
+ // let statement
+ consumeToken();
+ decompiler.addEOL(Token.LC);
+ nf.addChildToBack(result, statements(null));
+ mustMatchToken(Token.RC, "msg.no.curly.let");
+ decompiler.addToken(Token.RC);
+ } else {
+ // let expression
+ result.setType(Token.LETEXPR);
+ nf.addChildToBack(result, expr(false));
+ if (isStatement) {
+ // let expression in statement context
+ result = nf.createExprStatement(result, ts.getLineno());
+ }
+ }
+ } finally {
+ popScope();
+ }
+ return result;
+ }
+
+ void defineSymbol(int declType, boolean ignoreNotInBlock, String name) {
+ Node.Scope definingScope = currentScope.getDefiningScope(name);
+ Node.Scope.Symbol symbol = definingScope != null
+ ? definingScope.getSymbol(name)
+ : null;
+ boolean error = false;
+ if (symbol != null && (symbol.declType == Token.CONST ||
+ declType == Token.CONST))
+ {
+ error = true;
+ } else {
+ switch (declType) {
+ case Token.LET:
+ if (symbol != null && definingScope == currentScope) {
+ error = symbol.declType == Token.LET;
+ }
+ int currentScopeType = currentScope.getType();
+ if (!ignoreNotInBlock &&
+ ((currentScopeType == Token.LOOP) ||
+ (currentScopeType == Token.IF)))
+ {
+ addError("msg.let.decl.not.in.block");
+ }
+ currentScope.putSymbol(name,
+ new Node.Scope.Symbol(declType, name));
+ break;
+
+ case Token.VAR:
+ case Token.CONST:
+ case Token.FUNCTION:
+ if (symbol != null) {
+ if (symbol.declType == Token.VAR)
+ addStrictWarning("msg.var.redecl", name);
+ else if (symbol.declType == Token.LP) {
+ addStrictWarning("msg.var.hides.arg", name);
+ }
+ } else {
+ currentScriptOrFn.putSymbol(name,
+ new Node.Scope.Symbol(declType, name));
+ }
+ break;
+
+ case Token.LP:
+ if (symbol != null) {
+ // must be duplicate parameter. Second parameter hides the
+ // first, so go ahead and add the second pararameter
+ addWarning("msg.dup.parms", name);
+ }
+ currentScriptOrFn.putSymbol(name,
+ new Node.Scope.Symbol(declType, name));
+ break;
+
+ default:
+ throw Kit.codeBug();
+ }
+ }
+ if (error) {
+ addError(symbol.declType == Token.CONST ? "msg.const.redecl" :
+ symbol.declType == Token.LET ? "msg.let.redecl" :
+ symbol.declType == Token.VAR ? "msg.var.redecl" :
+ symbol.declType == Token.FUNCTION ? "msg.fn.redecl" :
+ "msg.parm.redecl", name);
+ }
}
private Node expr(boolean inForInit)
@@ -1311,6 +1506,9 @@ private Node expr(boolean inForInit)
decompiler.addToken(Token.COMMA);
if (compilerEnv.isStrictMode() && !pn.hasSideEffects())
addStrictWarning("msg.no.side.effects", "");
+ if (peekToken() == Token.YIELD) {
+ reportError("msg.yield.parenthesized");
+ }
pn = nf.createBinary(Token.COMMA, pn, assignExpr(inForInit));
}
return pn;
@@ -1319,9 +1517,14 @@ private Node expr(boolean inForInit)
private Node assignExpr(boolean inForInit)
throws IOException, ParserException
{
+ int tt = peekToken();
+ if (tt == Token.YIELD) {
+ consumeToken();
+ return returnOrYield(tt, true);
+ }
Node pn = condExpr(inForInit);
- int tt = peekToken();
+ tt = peekToken();
if (Token.FIRST_ASSIGN <= tt && tt <= Token.LAST_ASSIGN) {
consumeToken();
decompiler.addToken(tt);
@@ -1599,7 +1802,7 @@ private Node unaryExpr()
}
return pn;
}
- return nf.createName("err"); // Only reached on error. Try to continue.
+ return nf.createName("error"); // Only reached on error.Try to continue.
}
@@ -1682,6 +1885,9 @@ private void argumentList(Node listNode)
if (!first)
decompiler.addToken(Token.COMMA);
first = false;
+ if (peekToken() == Token.YIELD) {
+ reportError("msg.yield.parenthesized");
+ }
nf.addChildToBack(listNode, assignExpr(false));
} while (matchToken(Token.COMMA));
@@ -1764,6 +1970,13 @@ private Node memberExprTail(boolean allowCallSyntax, Node pn)
tt = nextToken();
switch (tt) {
+
+ // needed for generator.throw();
+ case Token.THROW:
+ decompiler.addName("throw");
+ pn = propertyName(pn, "throw", memberTypeFlags);
+ break;
+
// handles: name, ns::name, ns::*, ns::[expr]
case Token.NAME:
s = ts.getString();
@@ -1912,6 +2125,84 @@ private Node propertyName(Node pn, String name, int memberTypeFlags)
return pn;
}
+ private Node arrayComprehension(String arrayName, Node expr)
+ throws IOException, ParserException
+ {
+ if (nextToken() != Token.FOR)
+ throw Kit.codeBug(); // shouldn't be here if next token isn't 'for'
+ decompiler.addName(" "); // space after array literal expr
+ decompiler.addToken(Token.FOR);
+ boolean isForEach = false;
+ if (matchToken(Token.NAME)) {
+ decompiler.addName(ts.getString());
+ if (ts.getString().equals("each")) {
+ isForEach = true;
+ } else {
+ reportError("msg.no.paren.for");
+ }
+ }
+ mustMatchToken(Token.LP, "msg.no.paren.for");
+ decompiler.addToken(Token.LP);
+ String name;
+ int tt = peekToken();
+ if (tt == Token.LB || tt == Token.LC) {
+ // handle destructuring assignment
+ name = currentScriptOrFn.getNextTempName();
+ defineSymbol(Token.LP, false, name);
+ expr = nf.createBinary(Token.COMMA,
+ nf.createAssignment(Token.ASSIGN, primaryExpr(),
+ nf.createName(name)),
+ expr);
+ } else if (tt == Token.NAME) {
+ consumeToken();
+ name = ts.getString();
+ decompiler.addName(name);
+ } else {
+ reportError("msg.bad.var");
+ return nf.createNumber(0);
+ }
+
+ Node init = nf.createName(name);
+ // Define as a let since we want the scope of the variable to
+ // be restricted to the array comprehension
+ defineSymbol(Token.LET, false, name);
+
+ mustMatchToken(Token.IN, "msg.in.after.for.name");
+ decompiler.addToken(Token.IN);
+ Node iterator = expr(false);
+ mustMatchToken(Token.RP, "msg.no.paren.for.ctrl");
+ decompiler.addToken(Token.RP);
+
+ Node body;
+ tt = peekToken();
+ if (tt == Token.FOR) {
+ body = arrayComprehension(arrayName, expr);
+ } else {
+ Node call = nf.createCallOrNew(Token.CALL,
+ nf.createPropertyGet(nf.createName(arrayName), null,
+ "push", 0));
+ call.addChildToBack(expr);
+ body = new Node(Token.EXPR_VOID, call, ts.getLineno());
+ if (tt == Token.IF) {
+ consumeToken();
+ decompiler.addToken(Token.IF);
+ int lineno = ts.getLineno();
+ Node cond = condition();
+ body = nf.createIf(cond, body, null, lineno);
+ }
+ mustMatchToken(Token.RB, "msg.no.bracket.arg");
+ decompiler.addToken(Token.RB);
+ }
+
+ Node loop = enterLoop(null, true);
+ try {
+ return nf.createForIn(Token.LET, loop, init, iterator, body,
+ isForEach);
+ } finally {
+ exitLoop(false);
+ }
+ }
+
private Node primaryExpr()
throws IOException, ParserException
{
@@ -1928,6 +2219,7 @@ private Node primaryExpr()
case Token.LB: {
ObjArray elems = new ObjArray();
int skipCount = 0;
+ int destructuringLen = 0;
decompiler.addToken(Token.LB);
boolean after_lb_or_comma = true;
for (;;) {
@@ -1945,7 +2237,39 @@ private Node primaryExpr()
} else if (tt == Token.RB) {
consumeToken();
decompiler.addToken(Token.RB);
+ // for ([a,] in obj) is legal, but for ([a] in obj) is
+ // not since we have both key and value supplied. The
+ // trick is that [a,] and [a] are equivalent in other
+ // array literal contexts. So we calculate a special
+ // length value just for destructuring assignment.
+ destructuringLen = elems.size() +
+ (after_lb_or_comma ? 1 : 0);
break;
+ } else if (skipCount == 0 && elems.size() == 1 &&
+ tt == Token.FOR)
+ {
+ Node scopeNode = nf.createScopeNode(Token.ARRAYCOMP,
+ ts.getLineno());
+ String tempName = currentScriptOrFn.getNextTempName();
+ pushScope(scopeNode);
+ try {
+ defineSymbol(Token.LET, false, tempName);
+ Node expr = (Node) elems.get(0);
+ Node block = nf.createBlock(ts.getLineno());
+ Node init = new Node(Token.EXPR_VOID,
+ nf.createAssignment(Token.ASSIGN,
+ nf.createName(tempName),
+ nf.createCallOrNew(Token.NEW,
+ nf.createName("Array"))), ts.getLineno());
+ block.addChildToBack(init);
+ block.addChildToBack(arrayComprehension(tempName,
+ expr));
+ scopeNode.addChildToBack(block);
+ scopeNode.addChildToBack(nf.createName(tempName));
+ return scopeNode;
+ } finally {
+ popScope();
+ }
} else {
if (!after_lb_or_comma) {
reportError("msg.no.bracket.arg");
@@ -1954,7 +2278,7 @@ private Node primaryExpr()
after_lb_or_comma = false;
}
}
- return nf.createArrayLiteral(elems, skipCount);
+ return nf.createArrayLiteral(elems, skipCount, destructuringLen);
}
case Token.LC: {
@@ -2034,6 +2358,10 @@ private Node primaryExpr()
decompiler.addToken(Token.RC);
return nf.createObjectLiteral(elems);
}
+
+ case Token.LET:
+ decompiler.addToken(Token.LET);
+ return let(false);
case Token.LP:
View
311 src/org/mozilla/javascript/Token.java
@@ -1,29 +1,29 @@
/* ***** BEGIN LICENSE BLOCK *****
- *
- * Version: MPL 1.1
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
- * See the License for the specific language governing rights and
- * limitations under the License.
- *
- * The Original Code is org/mozilla/javascript/Token.java,
- * a component of the Rhino Library ( http://www.mozilla.org/rhino/ )
- * This file is a modification of the Original Code developed
- * for YUI Compressor.
- *
- * The Initial Developer of the Original Code is Mozilla Foundation
- *
- * Copyright (c) 2009 Mozilla Foundation. All Rights Reserved.
- *
- * Contributor(s): Yahoo! Inc. 2009
- *
- * ***** END LICENSE BLOCK ***** */
+*
+* Version: MPL 1.1
+*
+* The contents of this file are subject to the Mozilla Public License
+* Version 1.1 (the "License"); you may not use this file except in
+* compliance with the License. You may obtain a copy of the License
+* at http://www.mozilla.org/MPL/
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
+* See the License for the specific language governing rights and
+* limitations under the License.
+*
+* The Original Code is org/mozilla/javascript/Token.java,
+* a component of the Rhino Library ( http://www.mozilla.org/rhino/ )
+* This file is a modification of the Original Code developed
+* for YUI Compressor.
+*
+* The Initial Developer of the Original Code is Mozilla Foundation
+*
+* Copyright (c) 2009 Mozilla Foundation. All Rights Reserved.
+*
+* Contributor(s): Yahoo! Inc. 2009
+*
+* ***** END LICENSE BLOCK ***** */
package org.mozilla.javascript;
@@ -93,153 +93,161 @@
DELPROP = 31,
TYPEOF = 32,
GETPROP = 33,
- SETPROP = 34,
- GETELEM = 35,
- SETELEM = 36,
- CALL = 37,
- NAME = 38,
- NUMBER = 39,
- STRING = 40,
- NULL = 41,
- THIS = 42,
- FALSE = 43,
- TRUE = 44,
- SHEQ = 45, // shallow equality (===)
- SHNE = 46, // shallow inequality (!==)
- REGEXP = 47,
- BINDNAME = 48,
- THROW = 49,
- RETHROW = 50, // rethrow caught execetion: catch (e if ) use it
- IN = 51,
- INSTANCEOF = 52,
- LOCAL_LOAD = 53,
- GETVAR = 54,
- SETVAR = 55,
- CATCH_SCOPE = 56,
- ENUM_INIT_KEYS = 57,
- ENUM_INIT_VALUES = 58,
- ENUM_NEXT = 59,
- ENUM_ID = 60,
- THISFN = 61,
- RETURN_RESULT = 62, // to return prevoisly stored return result
- ARRAYLIT = 63, // array literal
- OBJECTLIT = 64, // object literal
- GET_REF = 65, // *reference
- SET_REF = 66, // *reference = something
- DEL_REF = 67, // delete reference
- REF_CALL = 68, // f(args) = something or f(args)++
- REF_SPECIAL = 69, // reference for special properties like __proto
+ GETPROPNOWARN = 34,
+ SETPROP = 35,
+ GETELEM = 36,
+ SETELEM = 37,
+ CALL = 38,
+ NAME = 39,
+ NUMBER = 40,
+ STRING = 41,
+ NULL = 42,
+ THIS = 43,
+ FALSE = 44,
+ TRUE = 45,
+ SHEQ = 46, // shallow equality (===)
+ SHNE = 47, // shallow inequality (!==)
+ REGEXP = 48,
+ BINDNAME = 49,
+ THROW = 50,
+ RETHROW = 51, // rethrow caught exception: catch (e if ) use it
+ IN = 52,
+ INSTANCEOF = 53,
+ LOCAL_LOAD = 54,
+ GETVAR = 55,
+ SETVAR = 56,
+ CATCH_SCOPE = 57,
+ ENUM_INIT_KEYS = 58,
+ ENUM_INIT_VALUES = 59,
+ ENUM_INIT_ARRAY= 60,
+ ENUM_NEXT = 61,
+ ENUM_ID = 62,
+ THISFN = 63,
+ RETURN_RESULT = 64, // to return previously stored return result
+ ARRAYLIT = 65, // array literal
+ OBJECTLIT = 66, // object literal
+ GET_REF = 67, // *reference
+ SET_REF = 68, // *reference = something
+ DEL_REF = 69, // delete reference
+ REF_CALL = 70, // f(args) = something or f(args)++
+ REF_SPECIAL = 71, // reference for special properties like __proto
+ YIELD = 72, // JS 1.7 yield pseudo keyword
// For XML support:
- DEFAULTNAMESPACE = 70, // default xml namespace =
- ESCXMLATTR = 71,
- ESCXMLTEXT = 72,
- REF_MEMBER = 73, // Reference for x.@y, x..y etc.
- REF_NS_MEMBER = 74, // Reference for x.ns::y, x..ns::y etc.
- REF_NAME = 75, // Reference for @y, @[y] etc.
- REF_NS_NAME = 76; // Reference for ns::y, @ns::y@[y] etc.
+ DEFAULTNAMESPACE = 73, // default xml namespace =
+ ESCXMLATTR = 74,
+ ESCXMLTEXT = 75,
+ REF_MEMBER = 76, // Reference for x.@y, x..y etc.
+ REF_NS_MEMBER = 77, // Reference for x.ns::y, x..ns::y etc.
+ REF_NAME = 78, // Reference for @y, @[y] etc.
+ REF_NS_NAME = 79; // Reference for ns::y, @ns::y@[y] etc.
// End of interpreter bytecodes
public final static int
LAST_BYTECODE_TOKEN = REF_NS_NAME,
- TRY = 77,
- SEMI = 78, // semicolon
- LB = 79, // left and right brackets
- RB = 80,
- LC = 81, // left and right curlies (braces)
- RC = 82,
- LP = 83, // left and right parentheses
- RP = 84,
- COMMA = 85, // comma operator
+ TRY = 80,
+ SEMI = 81, // semicolon
+ LB = 82, // left and right brackets
+ RB = 83,
+ LC = 84, // left and right curlies (braces)
+ RC = 85,
+ LP = 86, // left and right parentheses
+ RP = 87,
+ COMMA = 88, // comma operator
- ASSIGN = 86, // simple assignment (=)
- ASSIGN_BITOR = 87, // |=
- ASSIGN_BITXOR = 88, // ^=
- ASSIGN_BITAND = 89, // |=
- ASSIGN_LSH = 90, // <<=
- ASSIGN_RSH = 91, // >>=
- ASSIGN_URSH = 92, // >>>=
- ASSIGN_ADD = 93, // +=
- ASSIGN_SUB = 94, // -=
- ASSIGN_MUL = 95, // *=
- ASSIGN_DIV = 96, // /=
- ASSIGN_MOD = 97; // %=
+ ASSIGN = 89, // simple assignment (=)
+ ASSIGN_BITOR = 90, // |=
+ ASSIGN_BITXOR = 91, // ^=
+ ASSIGN_BITAND = 92, // |=
+ ASSIGN_LSH = 93, // <<=
+ ASSIGN_RSH = 94, // >>=
+ ASSIGN_URSH = 95, // >>>=
+ ASSIGN_ADD = 96, // +=
+ ASSIGN_SUB = 97, // -=
+ ASSIGN_MUL = 98, // *=
+ ASSIGN_DIV = 99, // /=
+ ASSIGN_MOD = 100; // %=
public final static int
FIRST_ASSIGN = ASSIGN,
LAST_ASSIGN = ASSIGN_MOD,
- HOOK = 98, // conditional (?:)
- COLON = 99,
- OR = 100, // logical or (||)
- AND = 101, // logical and (&&)
- INC = 102, // increment/decrement (++ --)
- DEC = 103,
- DOT = 104, // member operator (.)
- FUNCTION = 105, // function keyword
- EXPORT = 106, // export keyword
- IMPORT = 107, // import keyword
- IF = 108, // if keyword
- ELSE = 109, // else keyword
- SWITCH = 110, // switch keyword
- CASE = 111, // case keyword
- DEFAULT = 112, // default keyword
- WHILE = 113, // while keyword
- DO = 114, // do keyword
- FOR = 115, // for keyword
- BREAK = 116, // break keyword
- CONTINUE = 117, // continue keyword
- VAR = 118, // var keyword
- WITH = 119, // with keyword
- CATCH = 120, // catch keyword
- FINALLY = 121, // finally keyword
- VOID = 122, // void keyword
- RESERVED = 123, // reserved keywords
+ HOOK = 101, // conditional (?:)
+ COLON = 102,
+ OR = 103, // logical or (||)
+ AND = 104, // logical and (&&)
+ INC = 105, // increment/decrement (++ --)
+ DEC = 106,
+ DOT = 107, // member operator (.)
+ FUNCTION = 108, // function keyword
+ EXPORT = 109, // export keyword
+ IMPORT = 110, // import keyword
+ IF = 111, // if keyword
+ ELSE = 112, // else keyword
+ SWITCH = 113, // switch keyword
+ CASE = 114, // case keyword
+ DEFAULT = 115, // default keyword
+ WHILE = 116, // while keyword
+ DO = 117, // do keyword
+ FOR = 118, // for keyword
+ BREAK = 119, // break keyword
+ CONTINUE = 120, // continue keyword
+ VAR = 121, // var keyword
+ WITH = 122, // with keyword
+ CATCH = 123, // catch keyword
+ FINALLY = 124, // finally keyword
+ VOID = 125, // void keyword
+ RESERVED = 126, // reserved keywords
- EMPTY = 124,
+ EMPTY = 127,
/* types used for the parse tree - these never get returned
* by the scanner.
*/
- BLOCK = 125, // statement block
- LABEL = 126, // label
- TARGET = 127,
- LOOP = 128,
- EXPR_VOID = 129, // expression statement in functions
- EXPR_RESULT = 130, // expression statement in scripts
- JSR = 131,
- SCRIPT = 132, // top-level node for entire script
- TYPEOFNAME = 133, // for typeof(simple-name)
- USE_STACK = 134,
- SETPROP_OP = 135, // x.y op= something
- SETELEM_OP = 136, // x[y] op= something
- LOCAL_BLOCK = 137,
- SET_REF_OP = 138, // *reference op= something
+ BLOCK = 128, // statement block
+ LABEL = 129, // label
+ TARGET = 130,
+ LOOP = 131,
+ EXPR_VOID = 132, // expression statement in functions
+ EXPR_RESULT = 133, // expression statement in scripts
+ JSR = 134,
+ SCRIPT = 135, // top-level node for entire script
+ TYPEOFNAME = 136, // for typeof(simple-name)
+ USE_STACK = 137,
+ SETPROP_OP = 138, // x.y op= something
+ SETELEM_OP = 139, // x[y] op= something
+ LOCAL_BLOCK = 140,
+ SET_REF_OP = 141, // *reference op= something
// For XML support:
- DOTDOT = 139, // member operator (..)
- COLONCOLON = 140, // namespace::name
- XML = 141, // XML type
- DOTQUERY = 142, // .() -- e.g., x.emps.emp.(name == "terry")
- XMLATTR = 143, // @
- XMLEND = 144,
+ DOTDOT = 142, // member operator (..)
+ COLONCOLON = 143, // namespace::name
+ XML = 144, // XML type
+ DOTQUERY = 145, // .() -- e.g., x.emps.emp.(name == "terry")
+ XMLATTR = 146, // @
+ XMLEND = 147,
// Optimizer-only-tokens
- TO_OBJECT = 145,
- TO_DOUBLE = 146,
+ TO_OBJECT = 148,
+ TO_DOUBLE = 149,
- GET = 147, // JS 1.5 get pseudo keyword
- SET = 148, // JS 1.5 set pseudo keyword
- CONST = 149,
- SETCONST = 150,
- SETCONSTVAR = 151,
+ GET = 150, // JS 1.5 get pseudo keyword
+ SET = 151, // JS 1.5 set pseudo keyword
+ LET = 152, // JS 1.7 let pseudo keyword
+ CONST = 153,
+ SETCONST = 154,
+ SETCONSTVAR = 155,
+ ARRAYCOMP = 156, // array comprehension
+ LETEXPR = 157,
+ WITHEXPR = 158,
+ DEBUGGER = 159,
- CONDCOMMENT = 152, // JScript conditional comment
- KEEPCOMMENT = 153, // /*! ... */ comment
+ CONDCOMMENT = 160, // JScript conditional comment
+ KEEPCOMMENT = 161, // /*! ... */ comment
- LAST_TOKEN = 154;
+ LAST_TOKEN = 162;
public static String name(int token)
{
@@ -282,6 +290,7 @@ public static String name(int token)
case DELPROP: return "DELPROP";
case TYPEOF: return "TYPEOF";
case GETPROP: return "GETPROP";
+ case GETPROPNOWARN: return "GETPROPNOWARN";
case SETPROP: return "SETPROP";
case GETELEM: return "GETELEM";
case SETELEM: return "SETELEM";
@@ -306,7 +315,8 @@ public static String name(int token)
case SETVAR: return "SETVAR";
case CATCH_SCOPE: return "CATCH_SCOPE";
case ENUM_INIT_KEYS: return "ENUM_INIT_KEYS";
- case ENUM_INIT_VALUES: return "ENUM_INIT_VALUES";
+ case ENUM_INIT_VALUES:return "ENUM_INIT_VALUES";
+ case ENUM_INIT_ARRAY: return "ENUM_INIT_ARRAY";
case ENUM_NEXT: return "ENUM_NEXT";
case ENUM_ID: return "ENUM_ID";
case THISFN: return "THISFN";
@@ -370,6 +380,7 @@ public static String name(int token)
case WITH: return "WITH";
case CATCH: return "CATCH";
case FINALLY: return "FINALLY";
+ case VOID: return "VOID";
case RESERVED: return "RESERVED";
case EMPTY: return "EMPTY";
case BLOCK: return "BLOCK";
@@ -396,8 +407,14 @@ public static String name(int token)
case TO_DOUBLE: return "TO_DOUBLE";
case GET: return "GET";
case SET: return "SET";
+ case LET: return "LET";
+ case YIELD: return "YIELD";
case CONST: return "CONST";
case SETCONST: return "SETCONST";
+ case ARRAYCOMP: return "ARRAYCOMP";
+ case WITHEXPR: return "WITHEXPR";
+ case LETEXPR: return "LETEXPR";
+ case DEBUGGER: return "DEBUGGER";
}
// Token without name
View
134 src/org/mozilla/javascript/TokenStream.java
@@ -1,29 +1,29 @@
/* ***** BEGIN LICENSE BLOCK *****
- *
- * Version: MPL 1.1
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
- * See the License for the specific language governing rights and
- * limitations under the License.
- *
- * The Original Code is org/mozilla/javascript/TokenStream.java,
- * a component of the Rhino Library ( http://www.mozilla.org/rhino/ )
- * This file is a modification of the Original Code developed
- * for YUI Compressor.
- *
- * The Initial Developer of the Original Code is Mozilla Foundation
- *
- * Copyright (c) 2009 Mozilla Foundation. All Rights Reserved.
- *
- * Contributor(s): Yahoo! Inc. 2009
- *
- * ***** END LICENSE BLOCK ***** */
+*
+* Version: MPL 1.1
+*
+* The contents of this file are subject to the Mozilla Public License
+* Version 1.1 (the "License"); you may not use this file except in
+* compliance with the License. You may obtain a copy of the License
+* at http://www.mozilla.org/MPL/
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.
+* See the License for the specific language governing rights and
+* limitations under the License.
+*
+* The Original Code is org/mozilla/javascript/TokenStream.java,
+* a component of the Rhino Library ( http://www.mozilla.org/rhino/ )
+* This file is a modification of the Original Code developed
+* for YUI Compressor.
+*
+* The Initial Developer of the Original Code is Mozilla Foundation
+*
+* Copyright (c) 2009 Mozilla Foundation. All Rights Reserved.
+*
+* Contributor(s): Yahoo! Inc. 2009
+*
+* ***** END LICENSE BLOCK ***** */
package org.mozilla.javascript;
@@ -116,6 +116,7 @@ private static int stringToKeyword(String name)
Id_function = Token.FUNCTION,
Id_if = Token.IF,
Id_in = Token.IN,
+ Id_let = Token.LET,
Id_new = Token.NEW,
Id_null = Token.NULL,
Id_return = Token.RETURN,
@@ -127,6 +128,7 @@ private static int stringToKeyword(String name)
Id_void = Token.VOID,
Id_while = Token.WHILE,
Id_with = Token.WITH,
+ Id_yield = Token.YIELD,
// the following are #ifdef RESERVE_JAVA_KEYWORDS in jsscan.c
Id_abstract = Token.RESERVED,
@@ -136,7 +138,7 @@ private static int stringToKeyword(String name)
Id_char = Token.RESERVED,
Id_class = Token.RESERVED,
Id_const = Token.CONST,
- Id_debugger = Token.RESERVED,
+ Id_debugger = Token.DEBUGGER,
Id_double = Token.RESERVED,
Id_enum = Token.RESERVED,
Id_extends = Token.RESERVED,
@@ -178,6 +180,7 @@ private static int stringToKeyword(String name)
case 3: switch (s.charAt(0)) {
case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') {id=Id_for; break L0;} break L;
case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') {id=Id_int; break L0;} break L;
+ case 'l': if (s.charAt(2)=='t' && s.charAt(1)=='e') {id=Id_let; break L0;} break L;
case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') {id=Id_new; break L0;} break L;
case 't': if (s.charAt(2)=='y' && s.charAt(1)=='r') {id=Id_try; break L0;} break L;
case 'v': if (s.charAt(2)=='r' && s.charAt(1)=='a') {id=Id_var; break L0;} break L;
@@ -204,7 +207,10 @@ private static int stringToKeyword(String name)
} break L;
case 5: switch (s.charAt(2)) {
case 'a': X="class";id=Id_class; break L;
- case 'e': X="break";id=Id_break; break L;
+ case 'e': c=s.charAt(0);
+ if (c=='b') { X="break";id=Id_break; }
+ else if (c=='y') { X="yield";id=Id_yield; }
+ break L;
case 'i': X="while";id=Id_while; break L;
case 'l': X="false";id=Id_false; break L;
case 'n': c=s.charAt(0);
@@ -377,6 +383,14 @@ final int getToken() throws IOException
// Return the corresponding token if it's a keyword
int result = stringToKeyword(str);
if (result != Token.EOF) {
+ if ((result == Token.LET || result == Token.YIELD) &&
+ parser.compilerEnv.getLanguageVersion()
+ < Context.VERSION_1_7)
+ {
+ // LET and YIELD are tokens only in 1.7 and later
+ string = result == Token.LET ? "let" : "yield";
+ result = Token.NAME;
+ }
if (result != Token.RESERVED) {
return result;
} else if (!parser.compilerEnv.
@@ -631,9 +645,9 @@ final int getToken() throws IOException
skipLine();
continue retry;
}
- ungetChar('-');
+ ungetCharIgnoreLineEnd('-');
}
- ungetChar('!');
+ ungetCharIgnoreLineEnd('!');
}
if (matchChar('<')) {
if (matchChar('=')) {
@@ -703,10 +717,8 @@ final int getToken() throws IOException
String s1 = sb.toString();
String s2 = s1.trim();
if (s1.startsWith("!")) {
- // Remove the leading '!' ** EDIT actually don't remove it:
- // http://yuilibrary.com/projects/yuicompressor/ticket/2528008
- // this.string = s1.substring(1);
- this.string = s1;
+ // Remove the leading '!'
+ this.string = s1.substring(1);
return Token.KEEPCOMMENT;
} else if (s2.startsWith("@cc_on") ||
s2.startsWith("@if") ||
@@ -839,7 +851,6 @@ void readRegExp(int startToken)
} else if (c == ']') {
inClass = false;
}
-
addToString(c);
}
int reEnd = stringBufferTop;
@@ -1170,11 +1181,11 @@ private void ungetChar(int c)
private boolean matchChar(int test) throws IOException
{
- int c = getChar();
+ int c = getCharIgnoreLineEnd();
if (c == test) {
return true;
} else {
- ungetChar(c);
+ ungetCharIgnoreLineEnd(c);
return false;
}
}
@@ -1238,6 +1249,53 @@ private int getChar() throws IOException
}
}
+ private int getCharIgnoreLineEnd() throws IOException
+ {
+ if (ungetCursor != 0) {
+ return ungetBuffer[--ungetCursor];
+ }
+
+ for(;;) {
+ int c;
+ if (sourceString != null) {
+ if (sourceCursor == sourceEnd) {
+ hitEOF = true;
+ return EOF_CHAR;
+ }
+ c = sourceString.charAt(sourceCursor++);
+ } else {
+ if (sourceCursor == sourceEnd) {
+ if (!fillSourceBuffer()) {
+ hitEOF = true;
+ return EOF_CHAR;
+ }
+ }
+ c = sourceBuffer[sourceCursor++];
+ }
+
+ if (c <= 127) {
+ if (c == '\n' || c == '\r') {
+ lineEndChar = c;
+ c = '\n';
+ }
+ } else {
+ if (isJSFormatChar(c)) {
+ continue;
+ }
+ if (ScriptRuntime.isJSLineTerminator(c)) {
+ lineEndChar = c;
+ c = '\n';
+ }
+ }
+ return c;
+ }
+ }
+
+ private void ungetCharIgnoreLineEnd(int c)
+ {
+ ungetBuffer[ungetCursor++] = c;
+ }
+
private void skipLine() throws IOException
{
// skip to end of line
@@ -1329,8 +1387,8 @@ private boolean fillSourceBuffer() throws IOException
String regExpFlags;
- // Set this to an inital non-null value so that the Parser has
- // something to retrieve even if an error has occured and no
+ // Set this to an initial non-null value so that the Parser has
+ // something to retrieve even if an error has occurred and no
// string is found. Fosters one class of error, but saves lots of
// code.
private String string = "";
Please sign in to comment.
Something went wrong with that request. Please try again.