Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improved support for multibyte characters
- Loading branch information
Showing
5 changed files
with
55 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Original file line | Diff line number | Diff line change |
---|---|---|---|
@@ -1,32 +1,65 @@ | |||
# encoding: UTF-8 | |||
|
|||
require File.expand_path('../helper', __FILE__) | require File.expand_path('../helper', __FILE__) | ||
|
|
||
class MultibyteTest < Test::Unit::TestCase | class MultibyteTest < Test::Unit::TestCase | ||
grammar :Multibyte do | grammar :Multibyte do | ||
rule :string do | rule :string do | ||
"\xFF" | "ä" | ||
end | end | ||
|
|
||
rule :regexp do | rule :regexp do | ||
/\xFF/ | /(ä)+/ | ||
end | end | ||
|
|
||
rule :character_class do | rule :character_class do | ||
/[\xFF]/ | /[ä]+/ | ||
end | end | ||
end | end | ||
|
|
||
def test_multibyte_string | def test_multibyte_string | ||
m = Multibyte.parse("\xFF", :root => :string) | m = Multibyte.parse("ä", :root => :string) | ||
assert(m) | assert(m) | ||
end | end | ||
|
|
||
def test_multibyte_regexp | def test_multibyte_regexp | ||
m = Multibyte.parse("\xFF", :root => :regexp) | m = Multibyte.parse("äää", :root => :regexp) | ||
assert(m) | assert(m) | ||
end | end | ||
|
|
||
def test_multibyte_character_class | def test_multibyte_character_class | ||
m = Multibyte.parse("\xFF", :root => :character_class) | m = Multibyte.parse("äää", :root => :character_class) | ||
assert(m) | |||
end | |||
|
|||
Citrus.eval(<<-CITRUS) | |||
grammar Multibyte2 | |||
rule string | |||
"ä" | |||
end | |||
rule regexp | |||
/(ä)+/ | |||
end | |||
rule character_class | |||
[ä]+ | |||
end | |||
end | |||
CITRUS | |||
|
|||
def test_multibyte2_string | |||
m = Multibyte2.parse("ä", :root => :string) | |||
assert(m) | |||
end | |||
|
|||
def test_multibyte2_regexp | |||
m = Multibyte2.parse("äää", :root => :regexp) | |||
assert(m) | |||
end | |||
|
|||
def test_multibyte2_character_class | |||
m = Multibyte2.parse("äää", :root => :character_class) | |||
assert(m) | assert(m) | ||
end | end | ||
end | end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters