Various unicode related fixes (#226, #230, #251).

Merged pull-requests #252, #190 (manually).

.gitignore

@@ -9,11 +9,12 @@ __pycache__/
 __WORKDIR__/
 _build/
 _WORKSPACE/
+tools/virtualenvs
 .cache/
 .idea/
 .tox/
+.venv*/
 .DS_Store
 .coverage
 .ropeproject
 nosetests.xml
-tools/virtualenvs

CHANGES.rst

@@ -14,6 +14,7 @@ NEWS and CHANGES:
   - General:
 
     * Improve support for Python3 (py3.3, py3.4; #268)
+    * Various unicode related fixes (Unicode errors with non-ASCII, etc.)
     * Drop support for Python 2.5
 
   - Running:
@@ -39,7 +40,7 @@ ENHANCEMENTS:
   * pull #272: Use option role to format command line arg docs (provided by: helenst)
   * pull #271: Provide steps.catalog formatter (provided by: berdroid)
   * pull #261: Support "setup.cfg" as configuration file, too (provided by: bittner)
-  * pull #260: Documenation tweaks and typo fixes (provided by: bittner)
+  * pull #260: Documentation tweaks and typo fixes (provided by: bittner)
   * pull #254: Undefined step raises NotImplementedError instead of assert False (provided by: mhfrantz)
   * issue #242: JUnitReporter can show scenario tags (provided by: rigomes)
   * issue #240: Test Stages with different step implementations (provided by: attilammagyar, jenisys)
@@ -50,12 +51,19 @@ ENHANCEMENTS:
 FIXED:
 
   * pull #268: Fix py3 compatibility with all tests passed (provided by: sunliwen)
+  * pull #252: Related to #251 (provided by: mcepl)
+  * pull #190: UnicodeDecodeError in tracebacks (provided by: b3ni, vrutkovs, related to: #226, #230)
   * issue #257: Fix JUnitReporter (XML) for Python3 (provided by: actionless)
   * issue #249: Fix a number of docstring problems (provided by: masak)
   * issue #253: Various problems in PrettyFormatter.exception()
+  * issue #251: Unicode crash in model.py (provided by: mcepl, jenisys)
   * issue #236: Command line docs are confusing (solved by: #272)
+  * issue #230: problem with assert message that contains ascii over 128 value (provided by: jenisys)
+  * issue #226: UnicodeDecodeError in tracebacks (provided by: md1023, karulis, jenisys)
   * issue #221: Fix some PY2/PY3 incompatibilities (provided by: johbo)
   * issue #216: Using --wip option does not disable ANSI escape sequences (coloring).
+  * issue #119: Python3 support for behave (solved by: #268 and ...)
+  * issue #82:  JUnitReporter fails with Python 3.x (fixed with: #257, #268)
 
 
 .. _`New and Noteworthy`: https://github.com/behave/behave/blob/master/docs/new_and_noteworthy.rst

behave/__main__.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import, print_function
-import sys
 from behave import __version__
 from behave.configuration import Configuration, ConfigError
+from behave.parser import ParserError
 from behave.runner import Runner
 from behave.runner_util import print_undefined_step_snippets, \
     InvalidFileLocationError, InvalidFilenameError, FileNotFoundError
-from behave.parser import ParserError
+from behave.textutil import text as _text
+import sys
+
 
 TAG_HELP = """
 Scenarios inherit tags declared on the Feature level. The simplest
@@ -108,15 +110,20 @@ def main(args=None):
     try:
         failed = runner.run()
     except ParserError as e:
-        print("ParseError: %s" % e)
+        print(u"ParseError: %s" % e)
     except ConfigError as e:
-        print("ConfigError: %s" % e)
+        print(u"ConfigError: %s" % e)
     except FileNotFoundError as e:
-        print("FileNotFoundError: %s" % e)
+        print(u"FileNotFoundError: %s" % e)
     except InvalidFileLocationError as e:
-        print("InvalidFileLocationError: %s" % e)
+        print(u"InvalidFileLocationError: %s" % e)
     except InvalidFilenameError as e:
-        print("InvalidFilenameError: %s" % e)
+        print(u"InvalidFilenameError: %s" % e)
+    except Exception as e:
+        # -- DIAGNOSTICS:
+        text = _text(e)
+        print(u"Exception %s: %s" % (e.__class__.__name__, text))
+        raise
 
     if config.show_snippets and runner.undefined_steps:
         print_undefined_step_snippets(runner.undefined_steps,

behave/configuration.py

@@ -635,7 +635,7 @@ def build_name_re(names):
 
     def exclude(self, filename):
         if isinstance(filename, FileLocation):
-            filename = str(filename)
+            filename = six.text_type(filename)
 
         if self.include_re and self.include_re.search(filename) is None:
             return True

behave/formatter/formatters.py

@@ -2,7 +2,7 @@
 
 import sys
 from behave.formatter.base import StreamOpener
-from behave.textutil import compute_words_maxsize
+from behave.textutil import compute_words_maxsize, text as _text
 from behave.importer import LazyDict, LazyObject
 
 
@@ -33,7 +33,7 @@ def list_formatters(stream):
     """
     formatter_names = sorted(formatters)
     column_size = compute_words_maxsize(formatter_names)
-    schema = u"  %-"+ str(column_size) +"s  %s\n"
+    schema = u"  %-"+ _text(column_size) +"s  %s\n"
     for name in formatter_names:
         stream.write(schema % (name, formatters[name].description))
 

behave/formatter/pretty.py

@@ -4,7 +4,7 @@
 from behave.formatter.ansi_escapes import escapes, up
 from behave.formatter.base import Formatter
 from behave.model_describe import escape_cell, escape_triple_quotes
-from behave.textutil import indent
+from behave.textutil import indent, text as _text
 import sys
 import six
 from six.moves import range
@@ -211,11 +211,12 @@ def table(self, table):
 
     def doc_string(self, doc_string):
         #self.stream.write('      """' + doc_string.content_type + '\n')
-        prefix = '      '
-        self.stream.write('%s"""\n' % prefix)
+        doc_string = _text(doc_string)
+        prefix = u'      '
+        self.stream.write(u'%s"""\n' % prefix)
         doc_string = escape_triple_quotes(indent(doc_string, prefix))
         self.stream.write(doc_string)
-        self.stream.write('\n%s"""\n' % prefix)
+        self.stream.write(u'\n%s"""\n' % prefix)
         self.stream.flush()
 
     # def doc_string(self, doc_string):
@@ -227,7 +228,7 @@ def doc_string(self, doc_string):
 
     # -- UNUSED:
     # def exception(self, exception):
-    #     exception_text = str(exception)
+    #     exception_text = _text(exception)
     #     self.stream.write(self.format("failed").text(exception_text) + "\n")
     #     self.stream.flush()
 

behave/formatter/sphinx_util.py

@@ -4,7 +4,7 @@
 """
 
 from __future__ import absolute_import
-from behave.textutil import compute_words_maxsize
+from behave.textutil import compute_words_maxsize, text as _text
 import codecs
 import six
 
@@ -102,7 +102,7 @@ def write_table(self, table):
             column_size = compute_words_maxsize(column)
             cols_size.append(column_size)
             separator_parts.append("=" * column_size)
-            row_schema_parts.append("%-" + str(column_size) + "s")
+            row_schema_parts.append("%-" + _text(column_size) + "s")
 
         separator = " ".join(separator_parts) + "\n"
         row_schema = " ".join(row_schema_parts) + "\n"

behave/formatter/steps.py

@@ -7,7 +7,8 @@
 from __future__ import absolute_import
 from behave.formatter.base import Formatter
 from behave.step_registry import StepRegistry, registry
-from behave.textutil import compute_words_maxsize, indent, make_indentation
+from behave.textutil import \
+    compute_words_maxsize, indent, make_indentation, text as _text
 from behave import i18n
 from operator import attrgetter
 import inspect
@@ -170,7 +171,7 @@ def report_steps_by_type(self):
                 max_size = compute_words_maxsize(steps_text)
                 if max_size < self.min_location_column:
                     max_size = self.min_location_column
-                schema = u"  %-" + str(max_size) + "s  # %s\n"
+                schema = u"  %-" + _text(max_size) + "s  # %s\n"
             else:
                 schema = u"  %s\n"
 
@@ -269,14 +270,14 @@ class StepsCatalogFormatter(StepsDocFormatter):
     """
     Provides formatter class that shows the documentation of all registered
     step definitions. The primary purpose is to provide help for a test writer.
-    
+
     In order to ease work for non-programmer testers, the technical details of
     the steps (i.e. function name, source location) are ommited and the
     steps are shown as they would apprear in a feature file (no noisy '@',
     or '(', etc.).
-    
+
     Also, the output is sorted by step type (Given, When, Then)
-    
+
     Generic step definitions are listed with all three step types.
 
     EXAMPLE:
@@ -320,7 +321,7 @@ def describe_step_definition(self, step_definition, step_type=None):
                 desc.append(text)
         else:
             desc.append(u"%s %s" % (step_type.title(), step_definition.string))
-            
+
         return '\n'.join(desc)
 
 
@@ -431,9 +432,9 @@ def report_used_step_definitions(self):
             if max_size < self.min_location_column:
                 max_size = self.min_location_column
 
-            schema = u"%-" + str(max_size) + "s  # %s\n"
+            schema = u"%-" + _text(max_size) + "s  # %s\n"
             self.stream.write(schema % (stepdef_text, step_definition.location))
-            schema = u"%-" + str(max_size) + "s  # %s\n"
+            schema = u"%-" + _text(max_size) + "s  # %s\n"
             for step, step_text in zip(steps, steps_text):
                 self.stream.write(schema % (step_text, step.location))
             self.stream.write("\n")
@@ -455,10 +456,10 @@ def report_unused_step_definitions(self):
             max_size = self.min_location_column-2
 
         # -- STEP: Write report.
-        schema = u"  %-" + str(max_size) + "s  # %s\n"
+        schema = u"  %-" + _text(max_size) + "s  # %s\n"
         self.stream.write("UNUSED STEP DEFINITIONS[%d]:\n" % len(step_texts))
-        for step_definition, text in zip(step_definitions, step_texts):
-            self.stream.write(schema % (text, step_definition.location))
+        for step_definition, step_text in zip(step_definitions, step_texts):
+            self.stream.write(schema % (step_text, step_definition.location))
 
     def report_undefined_steps(self):
         if not self.undefined_steps:
@@ -475,7 +476,7 @@ def report_undefined_steps(self):
             max_size = self.min_location_column
 
         self.stream.write("\nUNDEFINED STEPS[%d]:\n" % len(steps_text))
-        schema = u"%-" + str(max_size) + "s  # %s\n"
+        schema = u"%-" + _text(max_size) + "s  # %s\n"
         for step, step_text in zip(undefined_steps, steps_text):
             self.stream.write(schema % (step_text, step.location))
 

behave/formatter/tags.py

@@ -9,7 +9,7 @@
 
 from __future__ import absolute_import
 from behave.formatter.base import Formatter
-from behave.textutil import compute_words_maxsize
+from behave.textutil import compute_words_maxsize, text as _text
 import six
 
 
@@ -121,7 +121,7 @@ def report_tag_counts(self):
         # -- PREPARE REPORT:
         ordered_tags = sorted(list(self.tag_counts.keys()))
         tag_maxsize = compute_words_maxsize(ordered_tags)
-        schema = "  @%-" + str(tag_maxsize) + "s %4d    (used for %s)\n"
+        schema = "  @%-" + _text(tag_maxsize) + "s %4d    (used for %s)\n"
 
         # -- EMIT REPORT:
         self.stream.write("TAG COUNTS (alphabetically sorted):\n")
@@ -138,7 +138,7 @@ def report_tag_counts_by_usage(self):
         ordered_tags = sorted(list(self.tag_counts.keys()),
                               key=compare_tag_counts_size)
         tag_maxsize = compute_words_maxsize(ordered_tags)
-        schema = "  @%-" + str(tag_maxsize) + "s %4d    (used for %s)\n"
+        schema = "  @%-" + _text(tag_maxsize) + "s %4d    (used for %s)\n"
 
         # -- EMIT REPORT:
         self.stream.write("TAG COUNTS (most often used first):\n")
@@ -176,7 +176,7 @@ def report_tags_by_locations(self):
         for tag_elements in self.tag_counts.values():
             locations.update([six.text_type(x.location) for x in tag_elements])
         location_column_size = compute_words_maxsize(locations)
-        schema = u"    %-" + str(location_column_size) + "s   %s\n"
+        schema = u"    %-" + _text(location_column_size) + "s   %s\n"
 
         # -- EMIT REPORT:
         self.stream.write("TAG LOCATIONS (alphabetically ordered):\n")