Add in a new Tokenizer.Iterator for String. Add a few other enhanceme…

…nts to make tokenizers and tokens easier to use

setup.winxed

@@ -228,6 +228,7 @@ function setup_stable_libraries(var rosella)
         "string/tokenizer/CClass",
         "string/tokenizer/Delimiter",
         "string/tokenizer/DelimiterRegion",
+        "string/tokenizer/Iterator",
         "string/tokenizer/Token"
     );
 

src/string/Tokenizer.winxed

@@ -146,5 +146,10 @@ namespace Rosella { namespace String
             self.data = substr(str, count + skip);
             return substr(str, 0, count);
         }
+
+        function get_iter[vtable]()
+        {
+            return new Rosella.String.Tokenizer.Iterator(self);
+        }
     }
 }}

src/string/tokenizer/Iterator.winxed

@@ -0,0 +1,34 @@
+namespace Rosella.String.Tokenizer
+{
+    class Iterator
+    {
+        var tokenizer;
+
+        function Iterator(var tokenizer)
+        {
+            self.tokenizer = tokenizer;
+        }
+
+        function set_integer_native[vtable](int direction)
+        {
+            // ignored
+        }
+
+        function get_bool[vtable]()
+        {
+            return self.tokenizer != null && self.tokenizer.has_tokens();
+        }
+
+        function shift_pmc[vtable]()
+        {
+            say("shift_pmc");
+            return self.tokenizer.get_token();
+        }
+
+        function shift_string[vtable]()
+        {
+            say("shift_string");
+            return self.tokenizer.get_token().data();
+        }
+    }
+}

src/string/tokenizer/Token.winxed

@@ -31,6 +31,11 @@ namespace Rosella { namespace String { namespace Tokenizer
             return self.data;
         }
 
+        function get_string[vtable]()
+        {
+            return string(self.data);
+        }
+
         // Get the metadata object;
         function metadata()
         {