Lots of tokens!

colomon · Sep 3, 2012 · fde2d1f · fde2d1f
1 parent f0797bf
commit fde2d1f
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 0 deletions.
diff --git a/lib/ISO_10303_21/Grammar.pm b/lib/ISO_10303_21/Grammar.pm
@@ -19,4 +19,16 @@ grammar ISO_10303_21::Grammar
     token keyword { <user_defined_keyword> | <standard_keyword> }
 
     token sign { '+' | '-' }
+    token integer { <sign>? <digit> <digit>* }
+    token real { <sign>? <digit>+ '.' <digit>* [ 'E' <sign>? <digit>+ ]? }
+
+    token non_q_char { <special> | <digit> | <space> | <lower> | <upper> }
+    # next one needs control directive too.
+    token string { "'" [ <non_q_char> | [<apostrophe> ** 2] | [<reverse_solidus> ** 2] ]* "'" }
+
+    token entity_instance_name { '#' <digit>+ }
+    token enumeration { '.' <upper> [ <upper> | <digit> ]* '.' }
+
+    token hex { <[0..9]> | <[A..F]> }
+    token binary { '"' <[0..3]> <hex>* '"' }
 }
diff --git a/t/02-tokens.t b/t/02-tokens.t
@@ -38,6 +38,36 @@ for ("+", "-") -> $keyword {
     ok $match, "<sign> matches $keyword - 2";
 }
 
+for <1 -1 +1 342 -3 0 -0> -> $keyword {
+    my $match = ISO_10303_21::Grammar.parse($keyword, :rule<integer>);
+    isa_ok $match, Match, "<integer> matches $keyword - 1";
+    ok $match, "<integer> matches $keyword - 2";
+}
+
+for <1. -1.0 +1. 342.E2 -3.0E-2 0. -0.> -> $keyword {
+    my $match = ISO_10303_21::Grammar.parse($keyword, :rule<real>);
+    isa_ok $match, Match, "<real> matches $keyword - 1";
+    ok $match, "<real> matches $keyword - 2";
+}
+
+for ("''", "'This is a test'", "''''", "'\\\\a different test'") -> $keyword {
+    my $match = ISO_10303_21::Grammar.parse($keyword, :rule<string>);
+    isa_ok $match, Match, "<string> matches $keyword - 1";
+    ok $match, "<string> matches $keyword - 2";
+}
+
+for ("#1", "#23", "#141123") -> $keyword {
+    my $match = ISO_10303_21::Grammar.parse($keyword, :rule<entity_instance_name>);
+    isa_ok $match, Match, "<entity_instance_name> matches $keyword - 1";
+    ok $match, "<entity_instance_name> matches $keyword - 2";
+}
+
+for (".A.", ".AD2.", ".GD.") -> $keyword {
+    my $match = ISO_10303_21::Grammar.parse($keyword, :rule<enumeration>);
+    isa_ok $match, Match, "<enumeration> matches $keyword - 1";
+    ok $match, "<enumeration> matches $keyword - 2";
+}
+
 
 
 done;