Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Need to update to enebo branch for this

git-svn-id: http://svn.codehaus.org/jruby/branches/enebo_node@2359 961051c9-f516-0410-bf72-c9f7e237a7b7
  • Loading branch information...
commit 835e7ce9728f4851734db487c7c2d2e8741d9594 1 parent 0b8a560
@enebo enebo authored
Showing with 7,967 additions and 7,357 deletions.
  1. +4 −0 src/org/jruby/ast/ArrayNode.java
  2. +15 −4 src/org/jruby/ast/BlockArgNode.java
  3. +2 −1  src/org/jruby/ast/CallNode.java
  4. +67 −0 src/org/jruby/ast/CommentNode.java
  5. +2 −1  src/org/jruby/ast/FCallNode.java
  6. +18 −4 src/org/jruby/ast/ListNode.java
  7. +9 −2 src/org/jruby/ast/LocalVarNode.java
  8. +107 −106 src/org/jruby/ast/NodeTypes.java
  9. +3 −2 src/org/jruby/ast/VCallNode.java
  10. +1 −1  src/org/jruby/evaluator/DefinedVisitor.java
  11. +3 −2 src/org/jruby/evaluator/EvaluationState.java
  12. +5 −2 src/org/jruby/lexer/yacc/HeredocTerm.java
  13. +53 −3 src/org/jruby/lexer/yacc/ISourcePosition.java
  14. +0 −4 src/org/jruby/lexer/yacc/LexerSource.java
  15. +267 −241 src/org/jruby/lexer/yacc/RubyYaccLexer.java
  16. +37 −8 src/org/jruby/lexer/yacc/SourcePosition.java
  17. +18 −18 src/org/jruby/lexer/yacc/StringTerm.java
  18. +1,680 −1,951 src/org/jruby/parser/DefaultRubyParser.java
  19. +1,153 −1,417 src/org/jruby/parser/DefaultRubyParser.y
  20. +133 −178 src/org/jruby/parser/ParserSupport.java
  21. +10 −0 src/org/jruby/parser/RubyParserResult.java
  22. +4 −1 src/org/jruby/parser/Tokens.java
  23. +3,350 −3,286 src/org/jruby/parser/YyTables.java
  24. +27 −0 src/org/jruby/util/IdUtil.java
  25. +999 −125 test/testPositions.rb
View
4 src/org/jruby/ast/ArrayNode.java
@@ -44,6 +44,10 @@
*/
public class ArrayNode extends ListNode implements ILiteralNode {
static final long serialVersionUID = 6279246130032958596L;
+
+ public ArrayNode(ISourcePosition position, Node firstNode) {
+ super(position, NodeTypes.ARRAYNODE, firstNode);
+ }
public ArrayNode(ISourcePosition position) {
super(position, NodeTypes.ARRAYNODE);
View
19 src/org/jruby/ast/BlockArgNode.java
@@ -32,6 +32,7 @@
import java.util.List;
+import org.jruby.ast.types.INameNode;
import org.jruby.ast.visitor.NodeVisitor;
import org.jruby.evaluator.Instruction;
import org.jruby.lexer.yacc.ISourcePosition;
@@ -45,14 +46,16 @@
* in this example c is a BlockArgNode
* @author jpetersen
*/
-public class BlockArgNode extends Node {
+public class BlockArgNode extends Node implements INameNode {
static final long serialVersionUID = 8374824536805365398L;
private final int count;
+ private String name;
- public BlockArgNode(ISourcePosition position, int count) {
+ public BlockArgNode(ISourcePosition position, int count, String name) {
super(position, NodeTypes.BLOCKARGNODE);
this.count = count;
+ this.name = name;
}
/**
@@ -70,9 +73,17 @@ public Instruction accept(NodeVisitor iVisitor) {
public int getCount() {
return count;
}
-
+
+ /**
+ * Get the name of this block argument
+ *
+ * @return it's name
+ */
+ public String getName() {
+ return name;
+ }
+
public List childNodes() {
return EMPTY_LIST;
}
-
}
View
3  src/org/jruby/ast/CallNode.java
@@ -34,6 +34,7 @@
import java.io.IOException;
import java.util.List;
+import org.jruby.ast.types.INameNode;
import org.jruby.ast.visitor.NodeVisitor;
import org.jruby.evaluator.Instruction;
import org.jruby.lexer.yacc.ISourcePosition;
@@ -43,7 +44,7 @@
*
* @author jpetersen
*/
-public final class CallNode extends Node {
+public final class CallNode extends Node implements INameNode {
static final long serialVersionUID = -1993752395320088525L;
private final Node receiverNode;
View
67 src/org/jruby/ast/CommentNode.java
@@ -0,0 +1,67 @@
+/***** BEGIN LICENSE BLOCK *****
+ * Version: CPL 1.0/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Common Public
+ * License Version 1.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at http://www.eclipse.org/legal/cpl-v10.html
+ *
+ * Software distributed under the License is distributed on an "AS
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * rights and limitations under the License.
+ *
+ * Copyright (C) 2006 Thomas E Enebo <enebo@acm.org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either of the GNU General Public License Version 2 or later (the "GPL"),
+ * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the CPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the CPL, the GPL or the LGPL.
+ ***** END LICENSE BLOCK *****/
+package org.jruby.ast;
+
+import java.util.List;
+
+import org.jruby.ast.visitor.NodeVisitor;
+import org.jruby.evaluator.Instruction;
+import org.jruby.lexer.yacc.ISourcePosition;
+
+/**
+ * Representation of a comment. Note that comments are not part of evaluation so you do
+ * not get the ability to visit this node as part of evaluation. In theory we could add
+ * this if we envisioned some wacky annotation system, but we have no crazy ideas yet.
+ *
+ */
+public class CommentNode extends Node {
+ private static final long serialVersionUID = -8304070370230933044L;
+
+ // Textual data (this is missing newline as a single line comment and =begin..=end
+ // comments do contain the intermediary newlines.
+ String content;
+
+ public CommentNode(ISourcePosition position, String content) {
+ super(position, NodeTypes.COMMENTNODE);
+
+ this.content = content;
+ }
+
+ public Instruction accept(NodeVisitor visitor) {
+ return null;
+ }
+
+ public String getContent() {
+ return content;
+ }
+
+ public List childNodes() {
+ return EMPTY_LIST;
+ }
+
+}
View
3  src/org/jruby/ast/FCallNode.java
@@ -34,6 +34,7 @@
import java.io.IOException;
import java.util.List;
+import org.jruby.ast.types.INameNode;
import org.jruby.ast.visitor.NodeVisitor;
import org.jruby.evaluator.Instruction;
import org.jruby.lexer.yacc.ISourcePosition;
@@ -42,7 +43,7 @@
*
* @author jpetersen
*/
-public class FCallNode extends Node {
+public class FCallNode extends Node implements INameNode {
static final long serialVersionUID = 3590332973770104094L;
private String name;
View
22 src/org/jruby/ast/ListNode.java
@@ -48,6 +48,18 @@
private List list = null;
+ /**
+ * Create a new ListNode.
+ *
+ * @param id type of listnode
+ * @param firstNode first element of the list
+ */
+ public ListNode(ISourcePosition position, int id, Node firstNode) {
+ this(position, id);
+
+ add(firstNode);
+ }
+
public ListNode(ISourcePosition position, int id) {
super(position, id);
}
@@ -57,10 +69,12 @@ public ListNode(ISourcePosition position) {
}
public ListNode add(Node node) {
- if (list == null) {
- list = new ArrayList();
- }
+ if (list == null) list = new ArrayList();
+ // Ruby Grammar productions return plenty of nulls.
+ if (node == null) return this;
+
list.add(node);
+ setPosition(getPosition().union(node.getPosition()));
return this;
}
@@ -105,7 +119,7 @@ public String toString() {
}
public List childNodes() {
- return list;
+ return list == null ? EMPTY_LIST : list;
}
public Instruction accept(NodeVisitor visitor) {
View
11 src/org/jruby/ast/LocalVarNode.java
@@ -32,6 +32,7 @@
import java.util.List;
+import org.jruby.ast.types.INameNode;
import org.jruby.ast.visitor.NodeVisitor;
import org.jruby.evaluator.Instruction;
import org.jruby.lexer.yacc.ISourcePosition;
@@ -40,14 +41,16 @@
*
* @author jpetersen
*/
-public class LocalVarNode extends Node {
+public class LocalVarNode extends Node implements INameNode {
static final long serialVersionUID = 8562701804939317217L;
private final int count;
+ private final String name;
- public LocalVarNode(ISourcePosition position, int count) {
+ public LocalVarNode(ISourcePosition position, int count, String name) {
super(position, NodeTypes.LOCALVARNODE);
this.count = count;
+ this.name = name;
}
/**
@@ -66,6 +69,10 @@ public int getCount() {
return count;
}
+ public String getName() {
+ return name;
+ }
+
public List childNodes() {
return EMPTY_LIST;
}
View
213 src/org/jruby/ast/NodeTypes.java
@@ -1,106 +1,107 @@
-package org.jruby.ast;
-
-public final class NodeTypes {
- public static final int ALIASNODE = 0;
- public static final int ANDNODE = 1;
- public static final int ARGSCATNODE = 2;
- public static final int ARGSNODE = 3;
- public static final int ARGUMENTNODE = 4;
- public static final int ARRAYNODE = 5;
- public static final int ASSIGNABLENODE = 6;
- public static final int BACKREFNODE = 7;
- public static final int BEGINNODE = 8;
- public static final int BIGNUMNODE = 9;
- public static final int BINARYOPERATORNODE = 10;
- public static final int BLOCKARGNODE = 11;
- public static final int BLOCKNODE = 12;
- public static final int BLOCKPASSNODE = 13;
- public static final int BREAKNODE = 14;
- public static final int CALLNODE = 15;
- public static final int CASENODE = 16;
- public static final int CLASSNODE = 17;
- public static final int CLASSVARASGNNODE = 18;
- public static final int CLASSVARDECLNODE = 19;
- public static final int CLASSVARNODE = 20;
- public static final int COLON2NODE = 21;
- public static final int COLON3NODE = 22;
- public static final int CONSTDECLNODE = 23;
- public static final int CONSTNODE = 24;
- public static final int DASGNNODE = 25;
- public static final int DEFINEDNODE = 26;
- public static final int DEFNNODE = 27;
- public static final int DEFSNODE = 28;
- public static final int DOTNODE = 29;
- public static final int DREGEXPNODE = 30;
- public static final int DSTRNODE = 31;
- public static final int DSYMBOLNODE = 32;
- public static final int DVARNODE = 33;
- public static final int DXSTRNODE = 34;
- public static final int ENSURENODE = 35;
- public static final int EVSTRNODE = 36;
- public static final int FALSENODE = 37;
- public static final int FCALLNODE = 38;
- public static final int FIXNUMNODE = 39;
- public static final int FLIPNODE = 40;
- public static final int FLOATNODE = 41;
- public static final int FORNODE = 42;
- public static final int GLOBALASGNNODE = 43;
- public static final int GLOBALVARNODE = 44;
- public static final int HASHNODE = 45;
- public static final int IFNODE = 46;
- public static final int INSTASGNNODE = 47;
- public static final int INSTVARNODE = 48;
- public static final int ISCOPINGNODE = 49;
- public static final int ITERNODE = 50;
- public static final int LISTNODE = 51;
- public static final int LOCALASGNNODE = 52;
- public static final int LOCALVARNODE = 53;
- public static final int MATCH2NODE = 54;
- public static final int MATCH3NODE = 55;
- public static final int MATCHNODE = 56;
- public static final int MODULENODE = 57;
- public static final int MULTIPLEASGNNODE = 58;
- public static final int NEWLINENODE = 59;
- public static final int NEXTNODE = 60;
- public static final int NILNODE = 61;
- public static final int NODETYPES = 62;
- public static final int NOTNODE = 63;
- public static final int NTHREFNODE = 64;
- public static final int OPASGNANDNODE = 65;
- public static final int OPASGNNODE = 66;
- public static final int OPASGNORNODE = 67;
- public static final int OPELEMENTASGNNODE = 68;
- public static final int OPTNNODE = 69;
- public static final int ORNODE = 70;
- public static final int POSTEXENODE = 71;
- public static final int REDONODE = 72;
- public static final int REGEXPNODE = 73;
- public static final int RESCUEBODYNODE = 74;
- public static final int RESCUENODE = 75;
- public static final int RETRYNODE = 76;
- public static final int RETURNNODE = 77;
- public static final int SCLASSNODE = 78;
- public static final int SCOPENODE = 79;
- public static final int SELFNODE = 80;
- public static final int SPLATNODE = 81;
- public static final int STARNODE = 82;
- public static final int STRNODE = 83;
- public static final int SUPERNODE = 84;
- public static final int SVALUENODE = 85;
- public static final int SYMBOLNODE = 86;
- public static final int TOARYNODE = 87;
- public static final int TRUENODE = 88;
- public static final int UNDEFNODE = 89;
- public static final int UNTILNODE = 90;
- public static final int VALIASNODE = 91;
- public static final int VCALLNODE = 92;
- public static final int WHENNODE = 93;
- public static final int WHILENODE = 94;
- public static final int XSTRNODE = 95;
- public static final int YIELDNODE = 96;
- public static final int ZARRAYNODE = 97;
- public static final int ZEROARGNODE = 98;
- public static final int ZSUPERNODE = 99;
-
- private NodeTypes() {}
-}
+package org.jruby.ast;
+
+public final class NodeTypes {
+ public static final int ALIASNODE = 0;
+ public static final int ANDNODE = 1;
+ public static final int ARGSCATNODE = 2;
+ public static final int ARGSNODE = 3;
+ public static final int ARGUMENTNODE = 4;
+ public static final int ARRAYNODE = 5;
+ public static final int ASSIGNABLENODE = 6;
+ public static final int BACKREFNODE = 7;
+ public static final int BEGINNODE = 8;
+ public static final int BIGNUMNODE = 9;
+ public static final int BINARYOPERATORNODE = 10;
+ public static final int BLOCKARGNODE = 11;
+ public static final int BLOCKNODE = 12;
+ public static final int BLOCKPASSNODE = 13;
+ public static final int BREAKNODE = 14;
+ public static final int CALLNODE = 15;
+ public static final int CASENODE = 16;
+ public static final int CLASSNODE = 17;
+ public static final int CLASSVARASGNNODE = 18;
+ public static final int CLASSVARDECLNODE = 19;
+ public static final int CLASSVARNODE = 20;
+ public static final int COLON2NODE = 21;
+ public static final int COLON3NODE = 22;
+ public static final int CONSTDECLNODE = 23;
+ public static final int CONSTNODE = 24;
+ public static final int DASGNNODE = 25;
+ public static final int DEFINEDNODE = 26;
+ public static final int DEFNNODE = 27;
+ public static final int DEFSNODE = 28;
+ public static final int DOTNODE = 29;
+ public static final int DREGEXPNODE = 30;
+ public static final int DSTRNODE = 31;
+ public static final int DSYMBOLNODE = 32;
+ public static final int DVARNODE = 33;
+ public static final int DXSTRNODE = 34;
+ public static final int ENSURENODE = 35;
+ public static final int EVSTRNODE = 36;
+ public static final int FALSENODE = 37;
+ public static final int FCALLNODE = 38;
+ public static final int FIXNUMNODE = 39;
+ public static final int FLIPNODE = 40;
+ public static final int FLOATNODE = 41;
+ public static final int FORNODE = 42;
+ public static final int GLOBALASGNNODE = 43;
+ public static final int GLOBALVARNODE = 44;
+ public static final int HASHNODE = 45;
+ public static final int IFNODE = 46;
+ public static final int INSTASGNNODE = 47;
+ public static final int INSTVARNODE = 48;
+ public static final int ISCOPINGNODE = 49;
+ public static final int ITERNODE = 50;
+ public static final int LISTNODE = 51;
+ public static final int LOCALASGNNODE = 52;
+ public static final int LOCALVARNODE = 53;
+ public static final int MATCH2NODE = 54;
+ public static final int MATCH3NODE = 55;
+ public static final int MATCHNODE = 56;
+ public static final int MODULENODE = 57;
+ public static final int MULTIPLEASGNNODE = 58;
+ public static final int NEWLINENODE = 59;
+ public static final int NEXTNODE = 60;
+ public static final int NILNODE = 61;
+ public static final int NODETYPES = 62;
+ public static final int NOTNODE = 63;
+ public static final int NTHREFNODE = 64;
+ public static final int OPASGNANDNODE = 65;
+ public static final int OPASGNNODE = 66;
+ public static final int OPASGNORNODE = 67;
+ public static final int OPELEMENTASGNNODE = 68;
+ public static final int OPTNNODE = 69;
+ public static final int ORNODE = 70;
+ public static final int POSTEXENODE = 71;
+ public static final int REDONODE = 72;
+ public static final int REGEXPNODE = 73;
+ public static final int RESCUEBODYNODE = 74;
+ public static final int RESCUENODE = 75;
+ public static final int RETRYNODE = 76;
+ public static final int RETURNNODE = 77;
+ public static final int SCLASSNODE = 78;
+ public static final int SCOPENODE = 79;
+ public static final int SELFNODE = 80;
+ public static final int SPLATNODE = 81;
+ public static final int STARNODE = 82;
+ public static final int STRNODE = 83;
+ public static final int SUPERNODE = 84;
+ public static final int SVALUENODE = 85;
+ public static final int SYMBOLNODE = 86;
+ public static final int TOARYNODE = 87;
+ public static final int TRUENODE = 88;
+ public static final int UNDEFNODE = 89;
+ public static final int UNTILNODE = 90;
+ public static final int VALIASNODE = 91;
+ public static final int VCALLNODE = 92;
+ public static final int WHENNODE = 93;
+ public static final int WHILENODE = 94;
+ public static final int XSTRNODE = 95;
+ public static final int YIELDNODE = 96;
+ public static final int ZARRAYNODE = 97;
+ public static final int ZEROARGNODE = 98;
+ public static final int ZSUPERNODE = 99;
+ public static final int COMMENTNODE = 100;
+
+ private NodeTypes() {}
+}
View
5 src/org/jruby/ast/VCallNode.java
@@ -34,6 +34,7 @@
import java.io.IOException;
import java.util.List;
+import org.jruby.ast.types.INameNode;
import org.jruby.ast.visitor.NodeVisitor;
import org.jruby.evaluator.Instruction;
import org.jruby.lexer.yacc.ISourcePosition;
@@ -43,7 +44,7 @@
*
* @author jpetersen
*/
-public class VCallNode extends Node {
+public class VCallNode extends Node implements INameNode {
static final long serialVersionUID = -7678578490000574578L;
private String name;
@@ -72,7 +73,7 @@ public Instruction accept(NodeVisitor iVisitor) {
* Gets the methodName.
* @return Returns a String
*/
- public String getMethodName() {
+ public String getName() {
return name;
}
View
2  src/org/jruby/evaluator/DefinedVisitor.java
@@ -195,7 +195,7 @@ public Instruction visitFCallNode(FCallNode iVisited) {
* @see AbstractVisitor#visitVCallNode(VCallNode)
*/
public Instruction visitVCallNode(VCallNode iVisited) {
- if (runtime.getCurrentContext().getFrameSelf().getMetaClass().isMethodBound(iVisited.getMethodName(), false)) {
+ if (runtime.getCurrentContext().getFrameSelf().getMetaClass().isMethodBound(iVisited.getName(), false)) {
definition = "method";
}
return null;
View
5 src/org/jruby/evaluator/EvaluationState.java
@@ -1107,7 +1107,8 @@ public static IRubyObject eval(ThreadContext context, Node node, IRubyObject sel
// If no exception is thrown execute else block
if (iVisited.getElseNode() != null) {
- if (iVisited.getRescueNode() == null) {
+ if (iVisited.getRescueNode() == null ||
+ iVisited.getRescueNode().getExceptionNodes().childNodes().isEmpty()) {
runtime.getWarnings().warn(iVisited.getElseNode().getPosition(), "else without rescue is useless");
}
result = eval(context, iVisited.getElseNode(), self);
@@ -1315,7 +1316,7 @@ public static IRubyObject eval(ThreadContext context, Node node, IRubyObject sel
}
case NodeTypes.VCALLNODE: {
VCallNode iVisited = (VCallNode) node;
- return self.callMethod(iVisited.getMethodName(), IRubyObject.NULL_ARRAY,
+ return self.callMethod(iVisited.getName(), IRubyObject.NULL_ARRAY,
CallType.VARIABLE);
}
case NodeTypes.WHENNODE:
View
7 src/org/jruby/lexer/yacc/HeredocTerm.java
@@ -27,6 +27,7 @@
***** END LICENSE BLOCK *****/
package org.jruby.lexer.yacc;
+import org.jruby.ast.StrNode;
import org.jruby.parser.Tokens;
@@ -89,8 +90,10 @@ public int parseString(RubyYaccLexer lexer, LexerSource src) {
case '$':
case '@':
src.unread(c);
+ lexer.setValue(new Token("#" + c, lexer.getPosition()));
return Tokens.tSTRING_DVAR;
case '{':
+ lexer.setValue(new Token("#" + c, lexer.getPosition()));
return Tokens.tSTRING_DBEG;
}
buffer.append('#');
@@ -103,7 +106,7 @@ public int parseString(RubyYaccLexer lexer, LexerSource src) {
throw new SyntaxException(src.getPosition(), "can't find string \"" + eos + "\" anywhere before EOF");
}
if (c != '\n') {
- lexer.yaccValue = new Token(buffer.toString(), lexer.getPosition(null, false));
+ lexer.yaccValue = new StrNode(lexer.getPosition(), buffer.toString());
return Tokens.tSTRING_CONTENT;
}
buffer.append(src.read());
@@ -119,7 +122,7 @@ public int parseString(RubyYaccLexer lexer, LexerSource src) {
src.unreadMany(lastLine);
lexer.setStrTerm(new StringTerm(-1, '\0', '\0'));
- lexer.yaccValue = new Token(str.toString(), lexer.getPosition(null, false));
+ lexer.yaccValue = new StrNode(lexer.getPosition(), str.toString());
return Tokens.tSTRING_CONTENT;
}
}
View
56 src/org/jruby/lexer/yacc/ISourcePosition.java
@@ -28,15 +28,65 @@
package org.jruby.lexer.yacc;
/**
- * This interface is the combination of two needs:
- * 1) A Ruby interpreter position (for warnings and errors).
- * 2) A IDE position (offsets for showing ranges of grammatical elements).
+ * This interface is the combination of two needs: 1) A Ruby interpreter position (for warnings
+ * and errors). The interpreter only cares about filename and endLine. 2) A IDE position (offsets
+ * for showing ranges of grammatical elements).
+ *
+ * The offsets represent positions 'between' characters. So a variable 'abc' has a startOffset
+ * of 0 and an endOffset of 3 ( 0a1b2c3 ).
*
*/
public interface ISourcePosition {
+ /**
+ * Which file does this source position live in?
+ *
+ * @return name of the source file.
+ */
public String getFile();
+
+ /**
+ * Which is the first(start) line that this source position occurs on (zero-based)
+ * @return
+ */
public int getStartLine();
+
+ /**
+ * Which is the last(end) line that this source position occurs on (zero-based)
+ *
+ * @return the line
+ */
public int getEndLine();
+
+ /**
+ * Modify startOffset by a relativeValue. At times our grammar and lexer do not give us
+ * the exact positions we need so we need to manually tweak position. The bummer of this
+ * is this requires ISourcePosition implementations are mutable.
+ *
+ * @param relativeValue to nudge startOffset up or down
+ */
+ public void adjustStartOffset(int relativeValue);
+
+ /**
+ * Get offset (relative to beginning of source file) immediately before first character
+ * represented by this source position.
+ *
+ * @return the offset
+ */
public int getStartOffset();
+
+ /**
+ * Get offset (relative to beginning of source file) immediately after the last character
+ * represented by this source position
+ *
+ * @return the offset
+ */
public int getEndOffset();
+
+ /**
+ * Calculates the logical union of the two positions and creates a new resulting position
+ *
+ * @param position to be unioned against this position
+ * @return a new position
+ */
+ public ISourcePosition union(ISourcePosition position);
}
View
4 src/org/jruby/lexer/yacc/LexerSource.java
@@ -240,10 +240,6 @@ public ISourcePosition getPosition() {
return positionFactory.getPosition(null, false);
}
- public ISourcePosition getDummyPosition() {
- return positionFactory.getDummyPosition();
- }
-
public ISourcePositionFactory getPositionFactory() {
return positionFactory;
}
View
508 src/org/jruby/lexer/yacc/RubyYaccLexer.java
@@ -14,10 +14,11 @@
* Copyright (C) 2002 Benoit Cerrina <b.cerrina@wanadoo.fr>
* Copyright (C) 2002-2004 Anders Bengtsson <ndrsbngtssn@yahoo.se>
* Copyright (C) 2002-2004 Jan Arne Petersen <jpetersen@uni-bonn.de>
- * Copyright (C) 2004 Thomas E Enebo <enebo@acm.org>
+ * Copyright (C) 2004-2006 Thomas E Enebo <enebo@acm.org>
* Copyright (C) 2004 Stefan Matthias Aust <sma@3plus4.de>
* Copyright (C) 2004-2005 David Corbin <dcorbin@users.sourceforge.net>
* Copyright (C) 2005 Zach Dennis <zdennis@mktec.com>
+ * Copyright (C) 2006 Thomas Corbat <tcorbat@hsr.ch>
*
* Alternatively, the contents of this file may be used under the terms of
* either of the GNU General Public License Version 2 or later (the "GPL"),
@@ -36,6 +37,10 @@
import java.math.BigInteger;
import org.jruby.ast.BackRefNode;
+import org.jruby.ast.BignumNode;
+import org.jruby.ast.CommentNode;
+import org.jruby.ast.FixnumNode;
+import org.jruby.ast.FloatNode;
import org.jruby.ast.NthRefNode;
import org.jruby.common.IRubyWarnings;
import org.jruby.parser.BlockNamesElement;
@@ -60,9 +65,6 @@
// Used for tiny smidgen of grammar in lexer (see setParserSupport())
private ParserSupport parserSupport = null;
- // The current location of the lexer immediately after a call to yylex()
- private ISourcePosition currentPos;
-
// What handles warnings
private IRubyWarnings warnings;
@@ -93,12 +95,9 @@
private final int str_squote = 0;
private final int str_dquote = STR_FUNC_EXPAND;
private final int str_xquote = STR_FUNC_EXPAND;
- private final int str_regexp =
- STR_FUNC_REGEXP|STR_FUNC_ESCAPE|STR_FUNC_EXPAND;
- //private final int str_sword = STR_FUNC_QWORDS;
- //private final int str_dword = STR_FUNC_QWORDS|STR_FUNC_EXPAND;
+ private final int str_regexp = STR_FUNC_REGEXP | STR_FUNC_ESCAPE | STR_FUNC_EXPAND;
private final int str_ssym = STR_FUNC_SYMBOL;
- private final int str_dsym = STR_FUNC_SYMBOL|STR_FUNC_EXPAND;
+ private final int str_dsym = STR_FUNC_SYMBOL | STR_FUNC_EXPAND;
public RubyYaccLexer() {
reset();
@@ -141,19 +140,6 @@ public Object value() {
return yaccValue;
}
- /**
- * Position from where last token came from.
- *
- * @return the last tokens position from the source
- */
- public ISourcePosition getPosition() {
- return currentPos;
- }
-
- public ISourcePosition getDummyPosition() {
- return src.getDummyPosition();
- }
-
public ISourcePositionFactory getPositionFactory() {
return src.getPositionFactory();
}
@@ -167,15 +153,11 @@ public ISourcePositionFactory getPositionFactory() {
* @return a new position
*/
public ISourcePosition getPosition(ISourcePosition startPosition, boolean inclusive) {
- return src.getPosition(startPosition, inclusive);
+ return src.getPosition(startPosition, inclusive);
}
- protected ISourcePosition getPositionMinusOne() {
- src.unread(' ');
- ISourcePosition position = getPosition(null, false);
- src.read();
-
- return position;
+ protected ISourcePosition getPosition() {
+ return src.getPosition(null, false);
}
/**
@@ -196,7 +178,6 @@ public void setParserSupport(ParserSupport parserSupport) {
*/
public void setSource(LexerSource source) {
this.src = source;
- currentPos = src.getDummyPosition();
}
public StrTerm getStrTerm() {
@@ -228,6 +209,10 @@ public StackState getCmdArgumentState() {
public StackState getConditionState() {
return conditionState;
}
+
+ public void setValue(Object yaccValue) {
+ this.yaccValue = yaccValue;
+ }
private boolean isNext_identchar() {
char c = src.read();
@@ -236,27 +221,23 @@ private boolean isNext_identchar() {
return c != EOF && (Character.isLetterOrDigit(c) || c == '-');
}
- private static final Object getInteger(String value, int radix) {
+ private Object getInteger(String value, int radix) {
try {
- return Long.valueOf(value, radix);
+ return new FixnumNode(getPosition(), Long.parseLong(value, radix));
} catch (NumberFormatException e) {
- return new BigInteger(value, radix);
+ return new BignumNode(getPosition(), new BigInteger(value, radix));
}
}
- private boolean ISUPPER(char c) {
- return Character.isUpperCase(c);
- }
-
/**
- * Do the next characters from the source match provided String in a case
- * insensitive manner. If so, then consume those characters and return
- * true. Otherwise, consume none of them.
+ * Do the next characters from the source match provided String in a case insensitive manner.
+ * If so, then consume those characters and that string. Otherwise, consume none of them and
+ * return null.
*
* @param s to be matched against
- * @return true if string matches
+ * @return string if string matches, null otherwise
*/
- private boolean isNextNoCase(String s) {
+ private String isNextNoCase(String s) {
StringBuffer buf = new StringBuffer();
for (int i = 0; i < s.length(); i++) {
@@ -267,11 +248,11 @@ private boolean isNextNoCase(String s) {
if (Character.toLowerCase(c) != r &&
Character.toUpperCase(c) != r) {
src.unreadMany(buf);
- return false;
+ return null;
}
}
- return true;
+ return buf.toString();
}
/**
@@ -298,72 +279,83 @@ private static final boolean isIdentifierChar(char c) {
return Character.isLetterOrDigit(c) || c == '_';
}
+ /**
+ * What type/kind of quote are we dealing with?
+ *
+ * @param c first character the the quote construct
+ * @return a token that specifies the quote type
+ */
private int parseQuote(char c) {
- char term;
- char paren;
+ char begin, end;
+ boolean shortHand;
+ // Short-hand (e.g. %{,%.,%!,... versus %Q{).
if (!Character.isLetterOrDigit(c)) {
- term = c;
+ begin = c;
c = 'Q';
+ shortHand = true;
+ // Long-hand (e.g. %Q{}).
} else {
- term = src.read();
- if (Character.isLetterOrDigit(term) /* no mb || ismbchar(term)*/) {
- throw new SyntaxException(src.getPosition(), "unknown type of %string");
+ shortHand = false;
+ begin = src.read();
+ if (Character.isLetterOrDigit(begin) /* no mb || ismbchar(term)*/) {
+ throw new SyntaxException(getPosition(), "unknown type of %string");
}
}
- if (c == EOF || term == EOF) {
- throw new SyntaxException(src.getPosition(), "unterminated quoted string meets end of file");
+ if (c == EOF || begin == EOF) {
+ throw new SyntaxException(getPosition(), "unterminated quoted string meets end of file");
}
- paren = term;
- if (term == '(') term = ')';
- else if (term == '[') term = ']';
- else if (term == '{') term = '}';
- else if (term == '<') term = '>';
- else paren = '\0';
+
+ // Figure end-char. '\0' is special to indicate begin=end and that no nesting?
+ if (begin == '(') end = ')';
+ else if (begin == '[') end = ']';
+ else if (begin == '{') end = '}';
+ else if (begin == '<') end = '>';
+ else { end = begin; begin = '\0'; };
switch (c) {
case 'Q':
- lex_strterm = new StringTerm(str_dquote, term, paren);
- yaccValue = new Token(""+c, getPositionMinusOne());
+ lex_strterm = new StringTerm(str_dquote, end, begin);
+ yaccValue = new Token("%"+ (shortHand ? (""+end) : ("" + c + begin)), getPosition());
return Tokens.tSTRING_BEG;
case 'q':
- lex_strterm = new StringTerm(str_squote, term, paren);
- yaccValue = new Token(""+c, getPositionMinusOne());
+ lex_strterm = new StringTerm(str_squote, end, begin);
+ yaccValue = new Token("%"+c+begin, getPosition());
return Tokens.tSTRING_BEG;
case 'W':
- lex_strterm = new StringTerm(str_dquote | STR_FUNC_QWORDS, term, paren);
+ lex_strterm = new StringTerm(str_dquote | STR_FUNC_QWORDS, end, begin);
do {c = src.read();} while (Character.isWhitespace(c));
src.unread(c);
- yaccValue = new Token(""+c, getPositionMinusOne());
+ yaccValue = new Token("%"+c+begin, getPosition());
return Tokens.tWORDS_BEG;
case 'w':
- lex_strterm = new StringTerm(str_squote | STR_FUNC_QWORDS, term, paren);
+ lex_strterm = new StringTerm(str_squote | STR_FUNC_QWORDS, end, begin);
do {c = src.read();} while (Character.isWhitespace(c));
src.unread(c);
- yaccValue = new Token(""+c, getPositionMinusOne());
+ yaccValue = new Token("%"+c+begin, getPosition());
return Tokens.tQWORDS_BEG;
case 'x':
- lex_strterm = new StringTerm(str_xquote, term, paren);
- yaccValue = new Token(""+c, getPositionMinusOne());
+ lex_strterm = new StringTerm(str_xquote, end, begin);
+ yaccValue = new Token("%"+c+begin, getPosition());
return Tokens.tXSTRING_BEG;
case 'r':
- lex_strterm = new StringTerm(str_regexp, term, paren);
- yaccValue = new Token(""+c, getPositionMinusOne());
+ lex_strterm = new StringTerm(str_regexp, end, begin);
+ yaccValue = new Token("%"+c+begin, getPosition());
return Tokens.tREGEXP_BEG;
case 's':
- lex_strterm = new StringTerm(str_ssym, term, paren);
+ lex_strterm = new StringTerm(str_ssym, end, begin);
lex_state = LexState.EXPR_FNAME;
- yaccValue = new Token(""+c, getPositionMinusOne());
+ yaccValue = new Token("%"+c+begin, getPosition());
return Tokens.tSYMBEG;
default:
- throw new SyntaxException(src.getPosition(), "Unknown type of %string. Expected 'Q', 'q', 'w', 'x', 'r' or any non letter character, but found '" + c + "'.");
+ throw new SyntaxException(getPosition(), "Unknown type of %string. Expected 'Q', 'q', 'w', 'x', 'r' or any non letter character, but found '" + c + "'.");
}
}
@@ -392,7 +384,7 @@ private int hereDocumentIdentifier() {
tokenBuffer.append(c);
}
if (c == EOF) {
- throw new SyntaxException(src.getPosition(), "unterminated here document identifier");
+ throw new SyntaxException(getPosition(), "unterminated here document identifier");
}
} else {
if (!isIdentifierChar(c)) {
@@ -415,14 +407,51 @@ private int hereDocumentIdentifier() {
String tok = tokenBuffer.toString();
lex_strterm = new HeredocTerm(tok, func, line);
- return term == '`' ? Tokens.tXSTRING_BEG : Tokens.tSTRING_BEG;
+ if (term == '`') {
+ yaccValue = new Token("`", getPosition());
+ return Tokens.tXSTRING_BEG;
+ }
+
+ yaccValue = new Token("\"", getPosition());
+ // Hacky: Advance position to eat newline here....
+ getPosition();
+ return Tokens.tSTRING_BEG;
}
private void arg_ambiguous() {
- warnings.warning(src.getPosition(), "Ambiguous first argument; make sure.");
+ warnings.warning(getPosition(), "Ambiguous first argument; make sure.");
}
+ /**
+ * Read a comment up to end of line. When found each comment will get stored away into
+ * the parser result so that any interested party can use them as they seem fit. One idea
+ * is that IDE authors can do distance based heuristics to associate these comments to the
+ * AST node they think they belong to.
+ *
+ * @param c last character read from lexer source
+ * @return newline or eof value
+ */
+ protected int readComment(char c) {
+ ISourcePosition startPosition = src.getPosition();
+ tokenBuffer.setLength(0);
+ tokenBuffer.append(c);
+ // FIXME: Consider making a better LexerSource.readLine
+ while ((c = src.read()) != '\n') {
+ tokenBuffer.append(c);
+ if (c == EOF) {
+ break;
+ }
+ }
+ src.unread(c);
+
+ // Store away each comment to parser result so IDEs can do whatever they want with them.
+ ISourcePosition position = startPosition.union(getPosition());
+ parserSupport.getResult().addComment(new CommentNode(position, tokenBuffer.toString()));
+
+ return c;
+ }
+
/*
* Not normally used, but is left in here since it can be useful in debugging
* grammar and lexing problems.
@@ -542,7 +571,7 @@ private int yylex() {
return token;
}
*/
-
+
/**
* Returns the next token. Also sets yyVal is needed.
*
@@ -562,8 +591,6 @@ private int yylex() {
return tok;
}
- currentPos = src.getPosition();
-
commandState = commandStart;
commandStart = false;
@@ -578,23 +605,20 @@ private int yylex() {
/* white spaces */
case ' ': case '\t': case '\f': case '\r':
case '\13': /* '\v' */
- getPosition(null, false);
+ getPosition();
spaceSeen = true;
continue retry;
case '#': /* it's a comment */
- while ((c = src.read()) != '\n') {
- if (c == EOF) {
- return 0;
- }
- }
-
+ if (readComment(c) == 0) return 0;
+
/* fall through */
case '\n':
// Replace a string of newlines with a single one
while((c = src.read()) == '\n') {
- currentPos = src.getPosition();
+
}
src.unread( c );
+ getPosition();
if (lex_state == LexState.EXPR_BEG ||
lex_state == LexState.EXPR_FNAME ||
@@ -610,22 +634,22 @@ private int yylex() {
case '*':
if ((c = src.read()) == '*') {
if ((c = src.read()) == '=') {
- yaccValue = new Token("**", getPosition(null, false));
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("**", getPosition());
return Tokens.tOP_ASGN;
}
src.unread(c);
- yaccValue = new Token("**", getPosition(null, false));
+ yaccValue = new Token("**", getPosition());
c = Tokens.tPOW;
} else {
if (c == '=') {
- yaccValue = new Token("*", getPosition(null, false));
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("*", getPosition());
return Tokens.tOP_ASGN;
}
src.unread(c);
if (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c)) {
- warnings.warning(src.getPosition(), "`*' interpreted as argument prefix");
+ warnings.warning(getPosition(), "`*' interpreted as argument prefix");
c = Tokens.tSTAR;
} else if (lex_state == LexState.EXPR_BEG ||
lex_state == LexState.EXPR_MID) {
@@ -640,25 +664,30 @@ private int yylex() {
} else {
lex_state = LexState.EXPR_BEG;
}
- yaccValue = new Token("*", getPosition(null, false));
+ yaccValue = new Token("*", getPosition());
return c;
case '!':
lex_state = LexState.EXPR_BEG;
if ((c = src.read()) == '=') {
- return Tokens.tNEQ;
+ yaccValue = new Token("!=",getPosition());
+ return Tokens.tNEQ;
}
if (c == '~') {
+ yaccValue = new Token("!~",getPosition());
return Tokens.tNMATCH;
}
src.unread(c);
+ yaccValue = new Token("!",getPosition());
return Tokens.tBANG;
case '=':
- // Skip documentation nodes
+ // documentation nodes
if (src.wasBeginOfLine()) {
- /* skip embedded rd document */
- if (isNextNoCase("begin")) {
+ String equalLabel;
+ if ((equalLabel = isNextNoCase("begin")) != null) {
+ tokenBuffer.setLength(0);
+ tokenBuffer.append(equalLabel);
c = src.read();
if (Character.isWhitespace(c)) {
@@ -666,38 +695,35 @@ private int yylex() {
src.unread(c);
for (;;) {
c = src.read();
+ tokenBuffer.append(c);
// If a line is followed by a blank line put
// it back.
while (c == '\n') {
c = src.read();
+ tokenBuffer.append(c);
}
if (c == EOF) {
- throw new SyntaxException(src.getPosition(), "embedded document meets end of file");
+ throw new SyntaxException(getPosition(), "embedded document meets end of file");
}
if (c != '=') continue;
- if (src.wasBeginOfLine() && isNextNoCase("end")) {
- //if (src.peek('\n')) {
- // break;
- //}
-
- //c = src.read();
-
- //if (Character.isWhitespace(c)) {
- src.readLine();
- break;
- //}
- //src.unread(c);
+ if (src.wasBeginOfLine() && (equalLabel = isNextNoCase("end")) != null) {
+ tokenBuffer.append(equalLabel);
+ tokenBuffer.append(src.readLine());
+ src.readLine();
+ src.unread('\n');
+ break;
}
}
+
+ parserSupport.getResult().addComment(new CommentNode(getPosition(), tokenBuffer.toString()));
continue retry;
}
src.unread(c);
}
}
- if (lex_state == LexState.EXPR_FNAME ||
- lex_state == LexState.EXPR_DOT) {
+ if (lex_state == LexState.EXPR_FNAME || lex_state == LexState.EXPR_DOT) {
lex_state = LexState.EXPR_ARG;
} else {
lex_state = LexState.EXPR_BEG;
@@ -707,22 +733,22 @@ private int yylex() {
if (c == '=') {
c = src.read();
if (c == '=') {
- yaccValue = new Token("===", getPosition(null, false));
+ yaccValue = new Token("===", getPosition());
return Tokens.tEQQ;
}
src.unread(c);
- yaccValue = new Token("==", getPosition(null, false));
+ yaccValue = new Token("==", getPosition());
return Tokens.tEQ;
}
if (c == '~') {
- yaccValue = new Token("=~", getPosition(null, false));
+ yaccValue = new Token("=~", getPosition());
return Tokens.tMATCH;
} else if (c == '>') {
- yaccValue = new Token("=>", getPosition(null, false));
+ yaccValue = new Token("=>", getPosition());
return Tokens.tASSOC;
}
src.unread(c);
- yaccValue = new Token("=", getPosition(null, false));
+ yaccValue = new Token("=", getPosition());
return '=';
case '<':
@@ -744,24 +770,24 @@ private int yylex() {
}
if (c == '=') {
if ((c = src.read()) == '>') {
- yaccValue = new Token("<=>", getPosition(null, false));
+ yaccValue = new Token("<=>", getPosition());
return Tokens.tCMP;
}
src.unread(c);
- yaccValue = new Token("<=", getPosition(null, false));
+ yaccValue = new Token("<=", getPosition());
return Tokens.tLEQ;
}
if (c == '<') {
if ((c = src.read()) == '=') {
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token("<<", getPosition(null, false));
+ yaccValue = new Token("<<", getPosition());
return Tokens.tOP_ASGN;
}
src.unread(c);
- yaccValue = new Token("<<", getPosition(null, false));
+ yaccValue = new Token("<<", getPosition());
return Tokens.tLSHFT;
}
- yaccValue = new Token("<", getPosition(null, false));
+ yaccValue = new Token("<", getPosition());
src.unread(c);
return Tokens.tLT;
@@ -774,29 +800,30 @@ private int yylex() {
}
if ((c = src.read()) == '=') {
- yaccValue = new Token(">=", getPosition(null, false));
+ yaccValue = new Token(">=", getPosition());
return Tokens.tGEQ;
}
if (c == '>') {
if ((c = src.read()) == '=') {
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token(">>", getPosition(null, false));
+ yaccValue = new Token(">>", getPosition());
return Tokens.tOP_ASGN;
}
src.unread(c);
- yaccValue = new Token(">>", getPosition(null, false));
+ yaccValue = new Token(">>", getPosition());
return Tokens.tRSHFT;
}
src.unread(c);
- yaccValue = new Token(">", getPosition(null, false));
+ yaccValue = new Token(">", getPosition());
return Tokens.tGT;
case '"':
lex_strterm = new StringTerm(str_dquote, '"', '\0');
+ yaccValue = new Token("\"", getPosition());
return Tokens.tSTRING_BEG;
case '`':
- yaccValue = new Token("`", getPosition(null, false));
+ yaccValue = new Token("`", getPosition());
if (lex_state == LexState.EXPR_FNAME) {
lex_state = LexState.EXPR_END;
return Tokens.tBACK_REF2;
@@ -814,17 +841,19 @@ private int yylex() {
case '\'':
lex_strterm = new StringTerm(str_squote, '\'', '\0');
+ yaccValue = new Token("'", getPosition());
return Tokens.tSTRING_BEG;
case '?':
if (lex_state == LexState.EXPR_END ||
lex_state == LexState.EXPR_ENDARG) {
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("?",getPosition());
return '?';
}
c = src.read();
if (c == EOF) {
- throw new SyntaxException(src.getPosition(), "incomplete character syntax");
+ throw new SyntaxException(getPosition(), "incomplete character syntax");
}
if (Character.isWhitespace(c)){
if (!lex_state.isArgument()){
@@ -852,12 +881,12 @@ private int yylex() {
break;
}
if (c2 != 0) {
- warnings.warn(src.getPosition(), "invalid character syntax; use ?\\" + c2);
+ warnings.warn(getPosition(), "invalid character syntax; use ?\\" + c2);
}
}
src.unread(c);
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token("?", getPosition(null, false));
+ yaccValue = new Token("?", getPosition());
return '?';
/*} else if (ismbchar(c)) { // ruby - we don't support them either?
rb_warn("multibyte character literal not supported yet; use ?\\" + c);
@@ -868,37 +897,36 @@ private int yylex() {
!src.peek('\n') && isNext_identchar()) {
src.unread(c);
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token("?", getPosition(null, false));
+ yaccValue = new Token("?", getPosition());
return '?';
} else if (c == '\\') {
c = src.readEscape();
}
c &= 0xff;
lex_state = LexState.EXPR_END;
- yaccValue = new Token(new Long(c), getPosition(null, false));
-
+ yaccValue = new FixnumNode(getPosition(), c);
return Tokens.tINTEGER;
case '&':
if ((c = src.read()) == '&') {
lex_state = LexState.EXPR_BEG;
if ((c = src.read()) == '=') {
- yaccValue = new Token("&&", getPosition(null, false));
+ yaccValue = new Token("&&", getPosition());
lex_state = LexState.EXPR_BEG;
return Tokens.tOP_ASGN;
}
src.unread(c);
- yaccValue = new Token("&&", getPosition(null, false));
+ yaccValue = new Token("&&", getPosition());
return Tokens.tANDOP;
}
else if (c == '=') {
- yaccValue = new Token("&", getPosition(null, false));
+ yaccValue = new Token("&", getPosition());
lex_state = LexState.EXPR_BEG;
return Tokens.tOP_ASGN;
}
src.unread(c);
if (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c)){
- warnings.warning(src.getPosition(), "`&' interpreted as argument prefix");
+ warnings.warning(getPosition(), "`&' interpreted as argument prefix");
c = Tokens.tAMPER;
} else if (lex_state == LexState.EXPR_BEG ||
lex_state == LexState.EXPR_MID) {
@@ -913,24 +941,24 @@ else if (c == '=') {
} else {
lex_state = LexState.EXPR_BEG;
}
- yaccValue = new Token("&", getPosition(null, false));
+ yaccValue = new Token("&", getPosition());
return c;
case '|':
if ((c = src.read()) == '|') {
lex_state = LexState.EXPR_BEG;
if ((c = src.read()) == '=') {
- yaccValue = new Token("||", getPosition(null, false));
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("||", getPosition());
return Tokens.tOP_ASGN;
}
src.unread(c);
- yaccValue = new Token("||", getPosition(null, false));
+ yaccValue = new Token("||", getPosition());
return Tokens.tOROP;
}
if (c == '=') {
- yaccValue = new Token("|", getPosition(null, false));
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("|", getPosition());
return Tokens.tOP_ASGN;
}
if (lex_state == LexState.EXPR_FNAME ||
@@ -940,7 +968,7 @@ else if (c == '=') {
lex_state = LexState.EXPR_BEG;
}
src.unread(c);
- yaccValue = new Token("|", getPosition(null, false));
+ yaccValue = new Token("|", getPosition());
return Tokens.tPIPE;
case '+':
@@ -949,16 +977,16 @@ else if (c == '=') {
lex_state == LexState.EXPR_DOT) {
lex_state = LexState.EXPR_ARG;
if (c == '@') {
- yaccValue = new Token("@+", getPosition(null, false));
+ yaccValue = new Token("+@", getPosition());
return Tokens.tUPLUS;
}
src.unread(c);
- yaccValue = new Token("+", getPosition(null, false));
+ yaccValue = new Token("+", getPosition());
return Tokens.tPLUS;
}
if (c == '=') {
- yaccValue = new Token("+", getPosition(null, false));
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("+", getPosition());
return Tokens.tOP_ASGN;
}
if (lex_state == LexState.EXPR_BEG ||
@@ -971,39 +999,37 @@ else if (c == '=') {
c = '+';
return parseNumber(c);
}
- yaccValue = new Token("+", getPosition(null, false));
+ yaccValue = new Token("+", getPosition());
return Tokens.tUPLUS;
}
lex_state = LexState.EXPR_BEG;
src.unread(c);
- yaccValue = new Token("+", getPosition(null, false));
+ yaccValue = new Token("+", getPosition());
return Tokens.tPLUS;
case '-':
c = src.read();
- if (lex_state == LexState.EXPR_FNAME ||
- lex_state == LexState.EXPR_DOT) {
+ if (lex_state == LexState.EXPR_FNAME || lex_state == LexState.EXPR_DOT) {
lex_state = LexState.EXPR_ARG;
if (c == '@') {
- yaccValue = new Token("@-", getPosition(null, false));
+ yaccValue = new Token("-@", getPosition());
return Tokens.tUMINUS;
}
src.unread(c);
- yaccValue = new Token("-", getPosition(null, false));
+ yaccValue = new Token("-", getPosition());
return Tokens.tMINUS;
}
if (c == '=') {
- yaccValue = new Token("-", getPosition(null, false));
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("-", getPosition());
return Tokens.tOP_ASGN;
}
- if (lex_state == LexState.EXPR_BEG ||
- lex_state == LexState.EXPR_MID ||
+ if (lex_state == LexState.EXPR_BEG || lex_state == LexState.EXPR_MID ||
(lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c))) {
if (lex_state.isArgument()) arg_ambiguous();
lex_state = LexState.EXPR_BEG;
src.unread(c);
- yaccValue = new Token("-", getPosition(null, false));
+ yaccValue = new Token("-", getPosition());
if (Character.isDigit(c)) {
return Tokens.tUMINUS_NUM;
}
@@ -1011,38 +1037,49 @@ else if (c == '=') {
}
lex_state = LexState.EXPR_BEG;
src.unread(c);
- yaccValue = new Token("-", getPosition(null, false));
+ yaccValue = new Token("-", getPosition());
return Tokens.tMINUS;
case '.':
lex_state = LexState.EXPR_BEG;
if ((c = src.read()) == '.') {
if ((c = src.read()) == '.') {
- yaccValue = new Token("...", getPosition(null, false));
+ yaccValue = new Token("...", getPosition());
return Tokens.tDOT3;
}
src.unread(c);
- yaccValue = new Token("..", getPosition(null, false));
+ yaccValue = new Token("..", getPosition());
return Tokens.tDOT2;
}
src.unread(c);
if (Character.isDigit(c)) {
- throw new SyntaxException(src.getPosition(), "no .<digit> floating literal anymore; put 0 before dot");
+ throw new SyntaxException(getPosition(), "no .<digit> floating literal anymore; put 0 before dot");
}
lex_state = LexState.EXPR_DOT;
- yaccValue = new Token(".", getPosition(null, false));
+ yaccValue = new Token(".", getPosition());
return Tokens.tDOT;
case '0' : case '1' : case '2' : case '3' : case '4' :
case '5' : case '6' : case '7' : case '8' : case '9' :
return parseNumber(c);
+ case ')':
+ conditionState.restart();
+ cmdArgumentState.restart();
+ lex_state = LexState.EXPR_END;
+ yaccValue = new Token(")", getPosition());
+ return Tokens.tRPAREN;
case ']':
+ conditionState.restart();
+ cmdArgumentState.restart();
+ lex_state = LexState.EXPR_END;
+ yaccValue = new Token(")", getPosition());
+ return Tokens.tRBRACK;
case '}':
- case ')':
conditionState.restart();
cmdArgumentState.restart();
lex_state = LexState.EXPR_END;
- return c;
+ yaccValue = new Token("}",getPosition());
+ return Tokens.tRCURLY;
case ':':
c = src.read();
@@ -1052,16 +1089,18 @@ else if (c == '=') {
lex_state == LexState.EXPR_CLASS ||
(lex_state.isArgument() && spaceSeen)) {
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token("::", getPosition(null, false));
+ yaccValue = new Token("::", getPosition());
return Tokens.tCOLON3;
}
lex_state = LexState.EXPR_DOT;
+ yaccValue = new Token(":",getPosition());
return Tokens.tCOLON2;
}
if (lex_state == LexState.EXPR_END ||
lex_state == LexState.EXPR_ENDARG || Character.isWhitespace(c)) {
src.unread(c);
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token(":",getPosition());
return ':';
}
switch (c) {
@@ -1076,18 +1115,19 @@ else if (c == '=') {
break;
}
lex_state = LexState.EXPR_FNAME;
- yaccValue = new Token(":", getPositionMinusOne());
+ yaccValue = new Token(":", getPosition());
return Tokens.tSYMBEG;
case '/':
if (lex_state == LexState.EXPR_BEG ||
lex_state == LexState.EXPR_MID) {
lex_strterm = new StringTerm(str_regexp, '/', '\0');
+ yaccValue = new Token("/",getPosition());
return Tokens.tREGEXP_BEG;
}
if ((c = src.read()) == '=') {
- yaccValue = new Token("/", getPositionMinusOne());
+ yaccValue = new Token("/", getPosition());
lex_state = LexState.EXPR_BEG;
return Tokens.tOP_ASGN;
}
@@ -1096,6 +1136,7 @@ else if (c == '=') {
if (!Character.isWhitespace(c)) {
arg_ambiguous();
lex_strterm = new StringTerm(str_regexp, '/', '\0');
+ yaccValue = new Token(Character.toString(c),getPosition());
return Tokens.tREGEXP_BEG;
}
}
@@ -1105,13 +1146,13 @@ else if (c == '=') {
} else {
lex_state = LexState.EXPR_BEG;
}
- yaccValue = new Token("/", getPosition(null, false));
+ yaccValue = new Token("/", getPosition());
return Tokens.tDIVIDE;
case '^':
if ((c = src.read()) == '=') {
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token("^", getPosition(null, false));
+ yaccValue = new Token("^", getPosition());
return Tokens.tOP_ASGN;
}
if (lex_state == LexState.EXPR_FNAME ||
@@ -1121,13 +1162,14 @@ else if (c == '=') {
lex_state = LexState.EXPR_BEG;
}
src.unread(c);
- yaccValue = new Token("^", getPosition(null, false));
+ yaccValue = new Token("^", getPosition());
return Tokens.tCARET;
case ';':
commandStart = true;
case ',':
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token(Character.toString(c), getPosition());
return c;
case '~':
@@ -1143,7 +1185,7 @@ else if (c == '=') {
} else {
lex_state = LexState.EXPR_BEG;
}
- yaccValue = new Token("~", getPosition(null, false));
+ yaccValue = new Token("~", getPosition());
return Tokens.tTILDE;
case '(':
c = Tokens.tLPAREN2;
@@ -1155,14 +1197,14 @@ else if (c == '=') {
if (lex_state == LexState.EXPR_CMDARG) {
c = Tokens.tLPAREN_ARG;
} else if (lex_state == LexState.EXPR_ARG) {
- warnings.warn(src.getPosition(), "don't put space before argument parentheses");
+ warnings.warn(getPosition(), "don't put space before argument parentheses");
c = Tokens.tLPAREN2;
}
}
conditionState.stop();
cmdArgumentState.stop();
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token("(", getPosition(null, false));
+ yaccValue = new Token("(", getPosition());
return c;
case '[':
@@ -1171,15 +1213,15 @@ else if (c == '=') {
lex_state = LexState.EXPR_ARG;
if ((c = src.read()) == ']') {
if ((c = src.read()) == '=') {
- yaccValue = new Token("[]=", getPosition(null, false));
+ yaccValue = new Token("[]=", getPosition());
return Tokens.tASET;
}
- yaccValue = new Token("[]", getPosition(null, false));
+ yaccValue = new Token("[]", getPosition());
src.unread(c);
return Tokens.tAREF;
}
src.unread(c);
- yaccValue = new Token("[", getPosition(null, false));
+ yaccValue = new Token("[", getPosition());
return '[';
} else if (lex_state == LexState.EXPR_BEG ||
lex_state == LexState.EXPR_MID) {
@@ -1190,7 +1232,7 @@ else if (c == '=') {
lex_state = LexState.EXPR_BEG;
conditionState.stop();
cmdArgumentState.stop();
- yaccValue = new Token("[", getPosition(null, false));
+ yaccValue = new Token("[", getPosition());
return c;
case '{':
@@ -1206,7 +1248,7 @@ else if (c == '=') {
conditionState.stop();
cmdArgumentState.stop();
lex_state = LexState.EXPR_BEG;
- yaccValue = new Token("{", getPosition(null, false));
+ yaccValue = new Token("{", getPosition());
return c;
case '\\':
@@ -1216,7 +1258,7 @@ else if (c == '=') {
continue retry; /* skip \\n */
}
src.unread(c);
- yaccValue = new Token("\\", getPosition(null, false));
+ yaccValue = new Token("\\", getPosition());
return '\\';
case '%':
@@ -1225,8 +1267,8 @@ else if (c == '=') {
return parseQuote(src.read());
}
if ((c = src.read()) == '=') {
- yaccValue = new Token("%", getPosition(null, false));
lex_state = LexState.EXPR_BEG;
+ yaccValue = new Token("%", getPosition());
return Tokens.tOP_ASGN;
}
if (lex_state.isArgument() && spaceSeen && !Character.isWhitespace(c)) {
@@ -1239,7 +1281,7 @@ else if (c == '=') {
lex_state = LexState.EXPR_BEG;
}
src.unread(c);
- yaccValue = new Token("%", getPosition(null, false));
+ yaccValue = new Token("%", getPosition());
return Tokens.tPERCENT;
case '$':
@@ -1274,7 +1316,7 @@ else if (c == '=') {
case '\"': /* $": already loaded files */
tokenBuffer.append('$');
tokenBuffer.append(c);
- yaccValue = new Token(tokenBuffer.toString(), getPosition(null, false));
+ yaccValue = new Token(tokenBuffer.toString(), getPosition());
return Tokens.tGVAR;
case '-':
@@ -1282,7 +1324,7 @@ else if (c == '=') {
tokenBuffer.append(c);
c = src.read();
tokenBuffer.append(c);
- yaccValue = new Token(tokenBuffer.toString(), getPosition(null, false));
+ yaccValue = new Token(tokenBuffer.toString(), getPosition());
/* xxx shouldn't check if valid option variable */
return Tokens.tGVAR;
@@ -1291,7 +1333,7 @@ else if (c == '=') {
case '`': /* $`: string before last match */
case '\'': /* $': string after last match */
case '+': /* $+: string matches last paren. */
- yaccValue = new BackRefNode(src.getPosition(), c);
+ yaccValue = new BackRefNode(getPosition(), c);
return Tokens.tBACK_REF;
case '1': case '2': case '3':
@@ -1303,13 +1345,14 @@ else if (c == '=') {
c = src.read();
} while (Character.isDigit(c));
src.unread(c);
- yaccValue = new NthRefNode(src.getPosition(), Integer.parseInt(tokenBuffer.substring(1)));
+ yaccValue = new NthRefNode(getPosition(), Integer.parseInt(tokenBuffer.substring(1)));
+
return Tokens.tNTH_REF;
default:
if (!isIdentifierChar(c)) {
src.unread(c);
- yaccValue = new Token("$", getPosition(null, false));
+ yaccValue = new Token("$", getPosition());
return '$';
}
case '0':
@@ -1327,13 +1370,13 @@ else if (c == '=') {
}
if (Character.isDigit(c)) {
if (tokenBuffer.length() == 1) {
- throw new SyntaxException(src.getPosition(), "`@" + c + "' is not allowed as an instance variable name");
+ throw new SyntaxException(getPosition(), "`@" + c + "' is not allowed as an instance variable name");
}
- throw new SyntaxException(src.getPosition(), "`@@" + c + "' is not allowed as a class variable name");
+ throw new SyntaxException(getPosition(), "`@@" + c + "' is not allowed as a class variable name");
}
if (!isIdentifierChar(c)) {
src.unread(c);
- yaccValue = new Token("@", getPosition(null, false));
+ yaccValue = new Token("@", getPosition());
return '@';
}
break;
@@ -1348,7 +1391,7 @@ else if (c == '=') {
default:
if (!isIdentifierChar(c)) {
- throw new SyntaxException(src.getPosition(), "Invalid char `\\" + new PrintfFormat("%.3o").sprintf(c) + "' in expression");
+ throw new SyntaxException(getPosition(), "Invalid char `\\" + new PrintfFormat("%.3o").sprintf(c) + "' in expression");
}
tokenBuffer.setLength(0);
@@ -1401,16 +1444,14 @@ else if (c == '=') {
result = Tokens.tFID;
} else {
if (lex_state == LexState.EXPR_FNAME) {
- /*
- // Enebo: This should be equivalent to below without
- // so much read/unread action.
if ((c = src.read()) == '=') {
char c2 = src.read();
if (c2 != '~' && c2 != '>' &&
(c2 != '=' || (c2 == '\n' && src.peek('>')))) {
- result = Token.tIDENTIFIER;
+ result = Tokens.tIDENTIFIER;
tokenBuffer.append(c);
+ src.unread(c2);
} else {
src.unread(c2);
src.unread(c);
@@ -1418,21 +1459,8 @@ else if (c == '=') {
} else {
src.unread(c);
}
- */
-
- if ((c = src.read()) == '=' &&
- !src.peek('~') &&
- !src.peek('>') &&
- (!src.peek('=') ||
- (src.peek('\n') &&
- src.getCharAt(1) == '>'))) {
- result = Tokens.tIDENTIFIER;
- tokenBuffer.append(c);
- } else {
- src.unread(c);
- }
}
- if (result == 0 && ISUPPER(tokenBuffer.charAt(0))) {
+ if (result == 0 && Character.isUpperCase(tokenBuffer.charAt(0))) {
result = Tokens.tCONSTANT;
} else {
result = Tokens.tIDENTIFIER;
@@ -1448,9 +1476,9 @@ else if (c == '=') {
lex_state = keyword.state;
if (state.isExprFName()) {
- yaccValue = new Token(keyword.name, getPositionMinusOne());
+ yaccValue = new Token(keyword.name, getPosition());
} else {
- yaccValue = new Token(tokenBuffer.toString(), getPositionMinusOne());
+ yaccValue = new Token(tokenBuffer.toString(), getPosition());
}
if (keyword.id0 == Tokens.kDO) {
if (conditionState.isInState()) {
@@ -1488,19 +1516,20 @@ else if (c == '=') {
} else {
lex_state = LexState.EXPR_END;
}
- }
- yaccValue = tokenBuffer.toString();
+ }
+
+ String tempVal = tokenBuffer.toString();
// Lame: parsing logic made it into lexer in ruby...So we
// are emulating
- if (IdUtil.isLocal((String)yaccValue) &&
+ if (IdUtil.getVarType(tempVal) != IdUtil.LOCAL_VAR &&
((((LocalNamesElement) parserSupport.getLocalNames().peek()).isInBlock() &&
- ((BlockNamesElement) parserSupport.getBlockNames().peek()).isDefined((String) yaccValue)) ||
- ((LocalNamesElement) parserSupport.getLocalNames().peek()).isLocalRegistered((String) yaccValue))) {
+ ((BlockNamesElement) parserSupport.getBlockNames().peek()).isDefined(tempVal)) ||
+ ((LocalNamesElement) parserSupport.getLocalNames().peek()).isLocalRegistered(tempVal))) {
lex_state = LexState.EXPR_END;
}
- yaccValue = new Token(yaccValue, getPositionMinusOne());
+ yaccValue = new Token(tempVal, getPosition());
return result;
}
@@ -1552,11 +1581,11 @@ private int parseNumber(char c) {
src.unread(c);
if (tokenBuffer.length() == startLen) {
- throw new SyntaxException(src.getPosition(), "Hexadecimal number without hex-digits.");
+ throw new SyntaxException(getPosition(), "Hexadecimal number without hex-digits.");
} else if (nondigit != '\0') {
- throw new SyntaxException(src.getPosition(), "Trailing '_' in number.");
+ throw new SyntaxException(getPosition(), "Trailing '_' in number.");
}
- yaccValue = new Token(getInteger(tokenBuffer.toString(), 16), getPositionMinusOne());
+ yaccValue = getInteger(tokenBuffer.toString(), 16);
return Tokens.tINTEGER;
case 'b' :
case 'B' : // binary
@@ -1579,11 +1608,11 @@ private int parseNumber(char c) {
src.unread(c);
if (tokenBuffer.length() == startLen) {
- throw new SyntaxException(src.getPosition(), "Binary number without digits.");
+ throw new SyntaxException(getPosition(), "Binary number without digits.");
} else if (nondigit != '\0') {
- throw new SyntaxException(src.getPosition(), "Trailing '_' in number.");
+ throw new SyntaxException(getPosition(), "Trailing '_' in number.");
}
- yaccValue = new Token(getInteger(tokenBuffer.toString(), 2), getPositionMinusOne());
+ yaccValue = getInteger(tokenBuffer.toString(), 2);
return Tokens.tINTEGER;
case 'd' :
case 'D' : // decimal
@@ -1606,11 +1635,11 @@ private int parseNumber(char c) {
src.unread(c);
if (tokenBuffer.length() == startLen) {
- throw new SyntaxException(src.getPosition(), "Binary number without digits.");
+ throw new SyntaxException(getPosition(), "Binary number without digits.");
} else if (nondigit != '\0') {
- throw new SyntaxException(src.getPosition(), "Trailing '_' in number.");
+ throw new SyntaxException(getPosition(), "Trailing '_' in number.");
}
- yaccValue = new Token(getInteger(tokenBuffer.toString(), 2), getPositionMinusOne());
+ yaccValue = getInteger(tokenBuffer.toString(), 2);
return Tokens.tINTEGER;
case '0' : case '1' : case '2' : case '3' : case '4' : //Octal
case '5' : case '6' : case '7' : case '_' :
@@ -1631,15 +1660,15 @@ private int parseNumber(char c) {
src.unread(c);
if (nondigit != '\0') {
- throw new SyntaxException(src.getPosition(), "Trailing '_' in number.");
+ throw new SyntaxException(getPosition(), "Trailing '_' in number.");
}
- yaccValue = new Token(getInteger(tokenBuffer.toString(), 8), getPositionMinusOne());
+ yaccValue = getInteger(tokenBuffer.toString(), 8);
return Tokens.tINTEGER;
}
case '8' :
case '9' :
- throw new SyntaxException(src.getPosition(), "Illegal octal digit.");
+ throw new SyntaxException(getPosition(), "Illegal octal digit.");
case '.' :
case 'e' :
case 'E' :
@@ -1647,7 +1676,7 @@ private int parseNumber(char c) {
break;
default :
src.unread(c);
- yaccValue = new Token(new Long(0), getPositionMinusOne());
+ yaccValue = new FixnumNode(getPosition(), 0);
return Tokens.tINTEGER;
}
}
@@ -1673,7 +1702,7 @@ private int parseNumber(char c) {
case '.' :
if (nondigit != '\0') {
src.unread(c);
- throw new SyntaxException(src.getPosition(), "Trailing '_' in number.");
+ throw new SyntaxException(getPosition(), "Trailing '_' in number.");
} else if (seen_point || seen_e) {
src.unread(c);