Allow overriding what function we use to convert to str (gvanrossum#19)

Always calling `str` causes trouble in the case where we are using
the codec to depyxlize python 2 code.

pyxl/codec/parser.py

@@ -18,7 +18,7 @@ def __init__(self, message, pos=None):
             super(ParseError, self).__init__(message)
 
 class PyxlParser(HTMLTokenizer):
-    def __init__(self, row, col):
+    def __init__(self, row, col, str_function):
         super(PyxlParser, self).__init__()
         self.start = self.end = (row, col)
         self.output = []
@@ -27,6 +27,7 @@ def __init__(self, row, col):
         self.next_thing_is_python = False
         self.last_thing_was_python = False
         self.last_thing_was_close_if_tag = False
+        self.str_function = str_function
 
     def delete_last_comma(self):
         for i in reversed(range(len(self.output))):
@@ -199,7 +200,7 @@ def format_parts():
             self.output.append('u"".join((')
             for part in attr_value:
                 if type(part) == list:
-                    self.output.append('str(')
+                    self.output.append('{}('.format(self.str_function))
                     self.output.append(Untokenizer().untokenize(part))
                     self.output.append(')')
                 else:

pyxl/codec/tokenizer.py

@@ -128,8 +128,8 @@ def pyxl_untokenize(tokens):
     return Untokenizer(1, 0).untokenize(tokens)
 
 
-def pyxl_tokenize(readline, invertible=False):
-    return cleanup_tokens(transform_tokens(RewindableTokenStream(readline), invertible))
+def pyxl_tokenize(readline, invertible=False, str_function='str'):
+    return cleanup_tokens(transform_tokens(RewindableTokenStream(readline), invertible, str_function))
 
 
 def pyxl_invert_tokenize(readline):
@@ -149,7 +149,7 @@ def cleanup_tokens(tokens):
         yield token
 
 
-def transform_tokens(tokens, invertible):
+def transform_tokens(tokens, invertible, str_function):
     last_nw_token = None
     prev_token = None
 
@@ -183,7 +183,7 @@ def transform_tokens(tokens, invertible):
              (last_nw_token[0] == tokenize.NAME and last_nw_token[1] == 'else') or
              (last_nw_token[0] == tokenize.NAME and last_nw_token[1] == 'yield') or
              (last_nw_token[0] == tokenize.NAME and last_nw_token[1] == 'return'))):
-            token = get_pyxl_token(token, tokens, invertible)
+            token = get_pyxl_token(token, tokens, invertible, str_function)
 
         if ttype not in (tokenize.INDENT,
                          tokenize.DEDENT,
@@ -205,9 +205,9 @@ def sanitize_token(token):
         return token
 
 
-def get_pyxl_token(start_token, tokens, invertible):
+def get_pyxl_token(start_token, tokens, invertible, str_function):
     ttype, tvalue, tstart, tend, tline = start_token
-    pyxl_parser = PyxlParser(tstart.row, tstart.col)
+    pyxl_parser = PyxlParser(tstart.row, tstart.col, str_function)
     pyxl_parser.feed(start_token)
 
     if invertible:
@@ -228,7 +228,7 @@ def get_pyxl_token(start_token, tokens, invertible):
                 division = get_end_pos(tstart, mid)
                 pyxl_parser.feed_position_only(Token(ttype, mid, tstart, division, tline))
                 tokens.rewind_and_retokenize(Token(ttype, right, division, tend, tline))
-                python_tokens = list(transform_tokens(tokens, invertible))
+                python_tokens = list(transform_tokens(tokens, invertible, str_function))
 
                 close_curly = next(tokens)
                 ttype, tvalue, tstart, tend, tline = close_curly

pyxl/codec/transform.py

@@ -7,9 +7,9 @@
     PyxlUnfinished,
 )
 
-def pyxl_transform(stream, invertible=False):
+def pyxl_transform(stream, invertible=False, str_function='str'):
     try:
-        output = pyxl_untokenize(pyxl_tokenize(stream.readline, invertible))
+        output = pyxl_untokenize(pyxl_tokenize(stream.readline, invertible, str_function))
     except Exception as ex:
         print(ex)
         traceback.print_exc()
@@ -31,9 +31,9 @@ def pyxl_invert(stream):
     return output
 
 
-def pyxl_transform_string(input, invertible=False):
+def pyxl_transform_string(input, invertible=False, str_function='str'):
     stream = io.StringIO(input)
-    return pyxl_transform(stream, invertible)
+    return pyxl_transform(stream, invertible, str_function)
 
 
 def pyxl_invert_string(input):