Permalink
Browse files

Made autotest file awesome.

Made block_append a little bit more accurate.
Added paren attrib to Sexp.
logop now walks the tree correctly, wasn't respecting parens too deep.
Fixed list_append.
Fixed evstr usage.
Fixed void_stmts' begin removals.
Hacked in \r handling... still broken.
compstmt prunes begins now.
Fixed to_ary.
Fixed op_asgn.
Fixed unary minus on lits.
Fixed trailing nil on for.
Fixed dxstr and dregx.

[git-p4: depot-paths = "//src/ruby_parser/dev/": change = 3701]
  • Loading branch information...
1 parent 417bb97 commit 4e94bf2bb0daf6b091193eb8e6953c51a0e4acd7 @zenspider zenspider committed Dec 12, 2007
Showing with 158 additions and 66 deletions.
  1. +10 −3 .autotest
  2. +37 −25 lib/ruby_lexer.rb
  3. +33 −18 lib/ruby_parser.y
  4. +11 −1 test/test_ruby_lexer.rb
  5. +67 −19 test/test_ruby_parser.rb
View
@@ -1,11 +1,18 @@
# -*- ruby -*-
Autotest.add_hook :initialize do |at|
+ at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
+ at.libs << ":../../ParseTree/dev/lib:../../ParseTree/dev/test"
+
at.unit_diff = "unit_diff -u -b"
- at.test_mappings[/^lib\/.*\.y$/] = proc { |filename, _|
- at.files_matching %r%^test/.*#{File.basename(filename, '.y').gsub '_', '_?'}.rb$%
- }
+ at.add_mapping(/^lib\/.*\.y$/) do |f, _|
+ at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
+ end
+
+ at.add_mapping(/pt_testcase.rb/) do |f, _|
+ at.files_matching(/^test.*rb$/)
+ end
end
Autotest.add_hook :run_command do |at|
View
@@ -198,7 +198,14 @@ def block_append(head, tail)
return head unless tail
return tail unless head
- head = head[1] if head[0] == :begin and head.size == 2 and tail[0] == :begin and $VERBOSE # HACK ugh... block_append two begins should be no/yes
+ case head[0]
+ when :lit, :str then
+ warn "unused literal ignored" # TODO: improve
+ return tail
+ end
+
+ head = head[1] if head[0] == :begin and head.size == 2 # TODO: remove_begin
+ # tail = tail[1] if tail[0] == :begin and tail.size == 2
head = s(:block, head) unless head[0] == :block
if Sexp === tail and tail[0] == :block then
@@ -222,17 +229,20 @@ def new_yield(node)
end
def logop(type, left, right)
- value_expr left
- if left && left[0] == type then
+ left = value_expr left
+
+ if left and left[0] == type and not left.paren then
node, second = left, nil
- while (second = node[2]) != 0 && second[0] == type do
+ while (second = node[2]) && second[0] == type and not second.paren do
node = second
end
node[2] = s(type, second, right)
+
return left
end
+
return s(type, left, right)
end
@@ -355,10 +365,9 @@ def arg_concat node1, node2
return node2.nil? ? node1 : s(:argscat, node1, node2)
end
- def list_append list, item
+ def list_append list, item # TODO: nuke me *sigh*
return s(:array, item) unless list
- list.push(*item[1..-1])
- list
+ list << item
end
def literal_concat head, tail
@@ -367,10 +376,7 @@ def literal_concat head, tail
htype, ttype = head[0], tail[0]
- if htype == :evstr then
- node = s(:dstr, '')
- head = list_append(node, head)
- end
+ head = s(:dstr, '', head) if htype == :evstr
case ttype
when :str then
@@ -385,14 +391,17 @@ def literal_concat head, tail
if htype == :str then
head[-1] << tail[-1]
else
- head << s(:str, tail[-1]) # return ((ListNode) head).addAll(tail);
+ tail[0] = :array
+ tail[1] = s(:str, tail[1])
+ head.push(*tail[1..-1])
end
when :evstr then
head[0] = :dstr if htype == :str
if tail[-1][0] == :str then # HACK FUCK YOU
head[-1] << tail[-1][-1]
+ head[0] = :str if head.size == 2 # HACK ?
else
- head = list_append(head, tail)
+ head.push(tail)
end
end
@@ -425,11 +434,7 @@ def void_stmts node # TODO: remove entirely... fuck it
return nil unless node
return node unless node[0] == :block
- while node and node[0] == :block and node.size == 2 do
- node = node[1]
- node = void_stmts(remove_begin(node))
- end
-
+ node[1..-2] = node[1..-2].map { |n| remove_begin(n) }
node
end
@@ -913,7 +918,7 @@ def heredoc here
if (func & RubyLexer::STR_FUNC_EXPAND) == 0 then
begin
- str << src.read_line
+ str << src.read_line.sub(/\r\n?|\n?\r/, "\n") # HACK
raise SyntaxError, err_msg if src.peek == RubyLexer::EOF
end until src.match_string(eosn, indent)
else
@@ -1816,10 +1821,10 @@ def yylex
token_buffer << '$'
token_buffer << '_'
- break if c =~ /\w/
-
- self.yacc_value = t(token_buffer.join)
- return :tGVAR
+ unless c =~ /\w/ then
+ self.yacc_value = t(token_buffer.join)
+ return :tGVAR
+ end
when /[~*$?!@\/\\;,.=:<>\"]/ then
token_buffer << '$'
token_buffer << c
@@ -2615,7 +2620,7 @@ def match_string term, indent=false # TODO: add case insensitivity, or just remo
if indent
while c = self.read do
- if c !~ /\s/ or c == "\n" then
+ if c !~ /\s/ or c == "\n" or c == "\r" then
self.unread c
break
end
@@ -2625,8 +2630,9 @@ def match_string term, indent=false # TODO: add case insensitivity, or just remo
term.each_byte do |c2|
c = self.read
+ c = self.read if c and c == "\r"
buffer << c
- if c2 != c[0] then
+ if c and c2 != c[0] then
self.unread_many buffer.join # HACK omg
return false
end
@@ -2672,6 +2678,12 @@ def unread_many str
end
class Sexp
+ attr_writer :paren
+
+ def paren
+ @paren ||= false
+ end
+
def value
raise "multi item sexp" if size > 2
last
View
@@ -68,7 +68,7 @@ bodystmt : compstmt opt_rescue opt_else opt_ensure {
}
compstmt : stmts opt_terms {
- result = val[0]
+ result = void_stmts(val[0])
}
stmts : none
@@ -157,22 +157,23 @@ stmt : kALIAS fitem { lexer.state = :expr_fname } fitem {
| mlhs '=' command_call {
val[2] = value_expr(val[2])
val[0][2] = if val[0][1] then
- s(:toary, val[2])
+ s(:to_ary, val[2])
else
s(:array, val[2])
end
result = val[0];
}
| var_lhs tOP_ASGN command_call {
- name = val[0].get_name;
- asgn_op = val[1].value;
+ name = val[0].last
+ asgn_op = val[1].value
val[2] = value_expr(val[2])
if asgn_op == "||" then
val[0][2] = (val[2]);
- result = s(:op_asgn, self.gettable(name), :"||", val[0])
+ result = s(:op_asgn_or, self.gettable(name), val[0])
elsif asgn_op == "&&" then
- result = s(:op_asgn, self.gettable(name), :"&&", val[0])
+ val[0][2] = (val[2]);
+ result = s(:op_asgn_and, self.gettable(name), val[0])
else
result = s(:lasgn,
self.gettable(name),
@@ -471,6 +472,8 @@ arg : lhs '=' arg {
name = val[0].value
asgn_op = val[1].value.to_sym
+ val[2] = remove_begin(val[2])
+
case asgn_op
when :"||" then
val[0] << val[2]
@@ -539,10 +542,10 @@ arg : lhs '=' arg {
result = s(:call, val[0], :**, s(:array, val[2]))
}
| tUMINUS_NUM tINTEGER tPOW arg {
- result = s(:call, s(:call, s(:lit, -val[1]), :"**", s(:array, val[3])), :"-@");
+ result = s(:call, s(:call, s(:lit, val[1]), :"**", s(:array, val[3])), :"-@");
}
| tUMINUS_NUM tFLOAT tPOW arg {
- result = s(:call, s(:call, s(:lit, -val[1]), :"**", s(:array, val[3])), :"-@");
+ result = s(:call, s(:call, s(:lit, val[1]), :"**", s(:array, val[3])), :"-@");
}
| tUPLUS arg {
if val[1][0] == :lit then
@@ -810,6 +813,7 @@ primary : literal
}
| tLPAREN compstmt tRPAREN {
result = val[1];
+ result.paren = true
}
| primary_value tCOLON2 tCONSTANT {
result = s(:colon2, val[0], val[2].value.to_sym)
@@ -951,7 +955,8 @@ primary : literal
} expr_value do {
lexer.cond.pop;
} compstmt kEND {
- result = s(:for, val[4], val[1], val[7])
+ result = s(:for, val[4], val[1])
+ result << val[7] if val[7]
}
| kCLASS cpath superclass {
if (self.in_def || self.in_single > 0) then
@@ -1149,10 +1154,10 @@ case_body : kWHEN when_args then compstmt cases {
when_args : args
| args ',' tSTAR arg_value {
result = self.list_append(val[0], s(:when, val[3], nil))
- }
+ }
| tSTAR arg_value {
result = s(:array, s(:when, val[1], nil));
- }
+ }
cases : opt_else | case_body
@@ -1204,7 +1209,7 @@ string : string1
string1 : tSTRING_BEG string_contents tSTRING_END {
result = val[1];
- extra_length = (val[0].value).length - 1;
+# extra_length = (val[0].value).length - 1;
# We may need to subtract addition offset off of first
# string fragment (we optimistically take one off in
@@ -1228,7 +1233,7 @@ xstring : tXSTRING_BEG xstring_contents tSTRING_END {
when :dstr
node[0] = :dxstr
else
- node = s(:dxstr, '', *node[1..-1])
+ node = s(:dxstr, '', node)
end
end
@@ -1296,7 +1301,7 @@ regexp : tREGEXP_BEG xstring_contents tREGEXP_END {
node[0] = :dregx
end
else
- node = s(:dregx, '', *node[1..-1]);
+ node = s(:dregx, '', node);
node[0] = :dregx_once if options =~ /o/
end
end
@@ -1315,7 +1320,8 @@ word_list : {
result = s(:array)
}
| word_list word ' ' {
- result = val[0].add(val[1][0] == :evstr ? s(:dstr, val[1]) : val[1])
+ word = val[1][0] == :evstr ? s(:dstr, '', val[1]) : val[1]
+ result = val[0] << word
}
word : string_content
@@ -1390,10 +1396,19 @@ sym : fname | tIVAR | tGVAR | tCVAR
dsym : tSYMBEG xstring_contents tSTRING_END {
lexer.state = :expr_end
- yyerror("empty symbol literal") if val[1].nil?
-
result = val[1]
- result[0] = :dsym if result[0] == :dstr
+
+ yyerror("empty symbol literal") if result.nil? or result.empty?
+
+ case result[0]
+ when :dstr then
+ result[0] = :dsym
+ when :str then
+ result = s(:lit, result.last.intern)
+ else
+ result = s(:dsym, '', result)
+ end
+
}
numeric : tINTEGER
@@ -213,6 +213,16 @@ def test_yylex_string_single_escapes
:tSTRING_END, t("'"))
end
+ def test_yylex_global
+ util_lex_token("$blah",
+ :tGVAR, t("$blah"))
+ end
+
+ def test_yylex_global_wierd
+ util_lex_token("$__blah",
+ :tGVAR, t("$__blah"))
+ end
+
def test_yylex_symbol
util_lex_token(":symbol",
:tSYMBEG, t(":"),
@@ -230,7 +240,7 @@ def util_lex_token input, *args
until args.empty? do
token = args.shift
value = args.shift
- assert @lex.advance
+ assert @lex.advance, "no more tokens"
assert_equal [token, value], [@lex.token, @lex.yacc_value]
end
Oops, something went wrong.

0 comments on commit 4e94bf2

Please sign in to comment.