Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

version 1.8.7: wrote a `UnicodeStream' class and test set for handlin…

…g non-ascii input data in Ruby 1.8
  • Loading branch information...
commit 77f7fe4f00009d8f7a98e8df17cfbabc1c7b8c8a 1 parent 8c992d1
Kyle Yetter authored
View
2  config/antlr3.yaml
@@ -3,7 +3,7 @@
# project meta-data used for various project tasks like building gem packages
#
name: antlr3
-version: "1.8.6"
+version: "1.8.7"
antlr_version: "3.2.1"
summary: |-
View
2  lib/antlr3/streams/unicode.rb
@@ -56,7 +56,7 @@ def look( k = 1 ) # for 1.8
def through( k )
if k >= 0 then @data[ @position, k ].pack( PACK_MASK ) else
start = ( @position + k ).at_least( 0 ) # start cannot be negative or index will wrap around
- @string[ start ... @position ].pack( PACK_MASK )
+ @data[ start ... @position ].pack( PACK_MASK )
end
end
View
2  lib/antlr3/version.rb
@@ -20,7 +20,7 @@ module ANTLR3
#
MAJOR_VERSION = 1
MINOR_VERSION = 8
- PATCH_VERSION = 6
+ PATCH_VERSION = 7
VERSION = [ MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION ]
VERSION_STRING = VERSION.join( '.' ).freeze
View
72 rakefile
@@ -73,63 +73,17 @@ task :boot => %w( update ) do
$project.load_environment
end
-# task :default => %w( setup )
-# $project.booted? or run_task( :setup )
+# tasks concerning the ANTLR java package
+desc "update the antlr jar if necessary"
+task :antlr do
+ $project.load_task 'antlr'
+end
-#if $project.booted?
-
- # tasks concerning the ANTLR java package
- desc "update the antlr jar if necessary"
- task :antlr do
- $project.load_task 'antlr'
- end
-
- for task_lib in %w( notes package coverage publish test doc )
- $project.load_task( task_lib )
- end
-
- desc( "clean up all project by-products" )
- task :clean => %w( test:clean package:clobber doc:clobber doc:samples:clobber )
-
- $project.load_task( 'private/*' )
-
- interactive = ( $stdout.tty? && $stdin.tty? )
- task_list = Rake.application.options.show_tasks
-
- #if $project.bundler_stale? and interactive and not task_list
- # puts( <<-END.here_indent! )
- # | ** Warning **
- # | Bundler config `#{ $project.bundler.config }' appears to have been modified recently.
- # | Your locally installed gems may be out of date and some tasks may
- # | fail due to missing packages.
- # |
- # | Do you want to run the setup task to update the local gem installations
- # | in the vendor directory before continuing? Note: this will access the
- # | internet and possibly download new source packages to ./vendor
- # |
- # END
- #
- # if $project.confirm( "Update before continuing?" )
- # Rake::Task[ :setup ].invoke
- # puts( "-- Successfully updated local gems --" )
- # else
- # puts( "-- Skipping bundler update --" )
- # puts( "** You can run the update manually later via `rake setup'" )
- # verbose( false ) { touch( $project.bundler.environment ) }
- # end
- # puts( "" )
- #
- #end
-
-#else
-#
-# puts( <<-END.here_indent! )
-# | ** Note **
-# | The development environment has not been established.
-# | Thus, most rake tasks will not be active until you run:
-# |
-# | ruby ./bootstrap.rb
-# |
-# END
-#
-##end
+for task_lib in %w( notes package coverage publish test doc )
+ $project.load_task( task_lib )
+end
+
+desc( "clean up all project by-products" )
+task :clean => %w( test:clean package:clobber doc:clobber doc:samples:clobber )
+
+$project.load_task( 'private/*' )
View
2  templates/Ruby.stg
@@ -1459,4 +1459,4 @@ placeAction(scope, name) ::= <<
<endif>
>>
-runtimeLibraryVersion() ::= "1.8.6"
+runtimeLibraryVersion() ::= "1.8.7"
View
56 test/unit/test-streams.rb
@@ -199,8 +199,6 @@ def test_rewind_nested
end
class TestFileStream < Test::Unit::TestCase
-
-
def test_no_encoding
path = File.join(File.dirname(__FILE__), 'sample-input/file-stream-1')
@@ -224,20 +222,11 @@ def test_no_encoding
@stream.peek(1).should == ?a.ord
end
- def test_encoded
-
- end
end
-class TestInputStream < Test::Unit::TestCase
- def test_no_encoding
-
- end
-
- def test_encoded
-
- end
-end
+
+
+
class TestCommonTokenStream < Test::Unit::TestCase
class MockSource
@@ -418,43 +407,4 @@ def new_token(type, opts = {})
end
-__END__
-teststreams.py | LN | STATUS
-----------------------------------------------+-----+--------------
-class TestStringStream(unittest.TestCase) | 009 | [x]
- def testSize(self) | 012 | [x]
- def testIndex(self) | 020 | [x]
- def testConsume(self) | 028 | [x]
- def testReset(self) | 079 | [x]
- def testLA(self) | 094 | [x]
- def testSubstring(self) | 111 | [x]
- def testSeekForward(self) | 122 | [x]
- def testMark(self) | 150 | [x]
- def testReleaseLast(self) | 167 | [x]
- def testReleaseNested(self) | 186 | [x]
- def testRewindLast(self) | 204 | [x]
- def testRewindNested(self) | 223 | [x]
-class TestFileStream(unittest.TestCase) | 245 | [o]
- def testNoEncoding(self) | 249 | [x]
- def testEncoded(self) | 272 | [ ]
-class TestInputStream(unittest.TestCase) | 296 | [ ]
- def testNoEncoding(self) | 299 | [ ]
- def testEncoded(self) | 322 | [ ]
-class TestCommonTokenStream(unittest.TestCase)| 345 | [ ]
- def setUp(self) | 348 | [x]
- def testInit(self) | 369 | [x]
- def testSetTokenSource(self) | 376 | [x]
- def testLTEmptySource(self) | 385 | [x]
- def testLT1(self) | 394 | [x]
- def testLT1WithHidden(self) | 407 | [x]
- def testLT2BeyondEnd(self) | 424 | [x]
- def testLTNegative(self) | 442 | [x]
- def testLB1(self) | 461 | [x]
- def testLTZero(self) | 479 | [x]
- def testLBBeyondBegin(self) | 496 | [x]
- def testFillBuffer(self) | 523 | [x]
- def testConsume(self) | 551 | [x]
- def testSeek(self) | 579 | [x]
- def testMarkRewind(self) | 604 | [x]
- def testToString(self) | 631 | [x]
View
186 test/unit/test-unicode-stream.rb
@@ -0,0 +1,186 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3'
+require 'antlr3/streams/unicode'
+
+require 'test/unit'
+require 'spec'
+
+
+include ANTLR3
+
+class TestUnicodeStream < Test::Unit::TestCase
+ def setup
+ # « a » \n 9 8 ° \n
+ # [ 171, 97, 187, 10, 57, 56, 176, 10]
+ @stream = UnicodeStream.new( "«a»\n98°\n" )
+ end
+
+ def test_size
+ @stream.size.should == 8
+ end
+
+ def test_index
+ @stream.index.should == 0
+ end
+
+ def test_consume
+ @stream.consume # «
+ @stream.index.should == 1
+ @stream.column.should == 1
+ @stream.line.should == 1
+
+ @stream.consume # a
+ @stream.index.should == 2
+ @stream.column.should == 2
+ @stream.line.should == 1
+
+ @stream.consume # »
+ @stream.index.should == 3
+ @stream.column.should == 3
+ @stream.line.should == 1
+
+ @stream.consume # \n
+ @stream.index.should == 4
+ @stream.column.should == 0
+ @stream.line.should == 2
+
+ @stream.consume # 9
+ @stream.index.should == 5
+ @stream.column.should == 1
+ @stream.line.should == 2
+
+ @stream.consume # 8
+ @stream.index.should == 6
+ @stream.column.should == 2
+ @stream.line.should == 2
+
+ @stream.consume # °
+ @stream.index.should == 7
+ @stream.column.should == 3
+ @stream.line.should == 2
+
+ @stream.consume # \n
+ @stream.index.should == 8
+ @stream.column.should == 0
+ @stream.line.should == 3
+
+ @stream.consume # EOF
+ @stream.index.should == 8
+ @stream.column.should == 0
+ @stream.line.should == 3
+
+ @stream.consume # EOF
+ @stream.index.should == 8
+ @stream.column.should == 0
+ @stream.line.should == 3
+ end
+
+ def test_reset
+ 2.times { @stream.consume }
+ @stream.reset
+ @stream.index.should == 0
+ @stream.line.should == 1
+ @stream.column.should == 0
+ @stream.peek(1).should == 171 # code point for «
+ end
+
+ def test_look
+ @stream.look(1).should == '«'
+ @stream.look(2).should == 'a'
+ @stream.look(3).should == "»"
+ @stream.peek(1).should == 171
+ @stream.peek(2).should == 97
+ @stream.peek(3).should == 187
+
+ 6.times { @stream.consume }
+ @stream.look(1).should == '°'
+ @stream.look(2).should == "\n"
+ @stream.look(3).should be_nil
+ @stream.peek(1).should == 176
+ @stream.peek(2).should == 10
+ @stream.peek(3).should == EOF
+ end
+
+ def test_substring
+ @stream.substring(0,0).should == '«'
+ @stream.substring(0,1).should == '«a'
+ @stream.substring(0,10).should == "«a»\n98°\n"
+ @stream.substring(3,6).should == "\n98°"
+ end
+
+ def test_seek_forward
+ @stream.seek(4)
+ @stream.index.should == 4
+ @stream.line.should == 2
+ @stream.column.should == 0
+ @stream.peek(1).should == 57
+ end
+
+ def test_mark
+ @stream.seek(4)
+ marker = @stream.mark
+ marker.should == 1
+
+ 2.times { @stream.consume }
+ marker = @stream.mark
+
+ marker.should == 2
+ end
+
+ def test_release_last
+ @stream.seek(4)
+ marker1 = @stream.mark
+
+ 2.times { @stream.consume }
+ marker2 = @stream.mark
+
+ @stream.release
+ @stream.mark_depth.should == 2
+ @stream.release
+ @stream.mark_depth.should == 1
+ end
+
+ def test_release_nested
+ @stream.seek(4)
+ marker1 = @stream.mark()
+
+ @stream.consume()
+ marker2 = @stream.mark()
+
+ @stream.consume()
+ marker3 = @stream.mark()
+
+ @stream.release(marker2)
+ @stream.mark_depth.should == 2
+
+ end
+
+ def test_rewind_last
+ @stream.seek(4)
+
+ marker = @stream.mark
+ @stream.consume
+ @stream.consume
+
+ @stream.rewind
+ @stream.mark_depth.should == 1
+ @stream.index.should == 4
+ @stream.line.should == 2
+ @stream.column.should == 0
+ @stream.peek(1).should == 57
+
+ end
+
+ def test_through
+ @stream.through( 2 ).should == '«a'
+ @stream.through( -2 ).should == ''
+ @stream.seek( 4 )
+ @stream.through( 0 ).should == ''
+ @stream.through( 1 ).should == '9'
+ @stream.through( -2 ).should == "»\n"
+ @stream.through( 5 ).should == "98°\n"
+ end
+end
+
Please sign in to comment.
Something went wrong with that request. Please try again.