Fix issue 830 - Make fails with UnicodeDecodeError

extras/help_generator/generate_help.py

@@ -28,6 +28,7 @@
 """
 
 import os
+import io
 import re
 import sys
 import textwrap
@@ -73,7 +74,7 @@
 # searching for a sli_command_list
 for file in allfiles:
     if file.endswith('.sli'):
-        f = open(file, 'r')
+        f = io.open(file, encoding='utf-8')
         filetext = f.read()
         f.close()
         items = re.findall(dcs, filetext, re.DOTALL)
@@ -93,7 +94,7 @@
 for fname in allfiles:
     # .py is for future use
     if not fname.endswith('.py'):
-        f = open(fname, 'r')
+        f = io.open(fname, encoding='utf-8')
         filetext = f.read()
         f.close()
         # Multiline matching to find codeblock

extras/help_generator/writers.py

@@ -27,6 +27,7 @@
 """
 
 import glob
+import io
 import os
 import re
 import textwrap
@@ -41,15 +42,15 @@ def write_help_html(doc_dic, helpdir, fname, sli_command_list, keywords):
     Write html for integration in NEST Help-System
     """
     # Loading Template for commands
-    ftemplate = open('templates/cmd.tpl.html', 'r')
+    ftemplate = io.open('templates/cmd.tpl.html', encoding='utf-8')
     templ = ftemplate.read()
     ftemplate.close()
     # Loading Template for CSS
-    cssf = open('templates/nest.tpl.css', 'r')
+    cssf = io.open('templates/nest.tpl.css', encoding='utf-8')
     csstempl = cssf.read()
     cssf.close()
     # Loading Template for footer
-    footerf = open('templates/footer.tpl.html', 'r')
+    footerf = io.open('templates/footer.tpl.html', encoding='utf-8')
     footertempl = footerf.read()
     footerf.close()
 
@@ -134,12 +135,13 @@ def write_help_html(doc_dic, helpdir, fname, sli_command_list, keywords):
         else:
             path = os.path.join(helpdir, 'cc')
 
-        f_file_name = open('{}/{}.html'.format(path, name), 'w')
-        "{}/{}.html".format(path, name)
+        f_file_name = io.open(os.path.join(path, '{}.html'.format(name)),
+                              mode='w', encoding='utf-8')
         f_file_name.write(cmdindexstring)
         f_file_name.close()
 
-        f_file_name_hlp = open('{}/{}.hlp'.format(path, name), 'w')
+        f_file_name_hlp = io.open(os.path.join(path, '{}.hlp'.format(name)),
+                                  mode='w', encoding='utf-8')
         f_file_name_hlp.write('\n'.join(hlplist))
         f_file_name_hlp.close()
 
@@ -155,15 +157,18 @@ def write_helpindex(helpdir):
     hlp_list = []
 
     # Loading Template for helpindex.html
-    ftemplate = open(os.path.join('templates', 'helpindex.tpl.html'), 'r')
+    ftemplate = io.open(os.path.join('templates', 'helpindex.tpl.html'),
+                        encoding='utf-8')
     templ = ftemplate.read()
     ftemplate.close()
     # Loading Template for CSS
-    cssf = open(os.path.join('templates', 'nest.tpl.css'), 'r')
+    cssf = io.open(os.path.join('templates', 'nest.tpl.css'),
+                   encoding='utf-8')
     csstempl = cssf.read()
     cssf.close()
     # Loading Template for footer
-    footerf = open(os.path.join('templates', 'footer.tpl.html'), 'r')
+    footerf = io.open(os.path.join('templates', 'footer.tpl.html'),
+                      encoding='utf-8')
     footertempl = footerf.read()
     footerf.close()
 
@@ -186,7 +191,7 @@ def write_helpindex(helpdir):
                          % doubles[0])
         for item in sorted(filelist,
                            key=lambda name: name.lower().rsplit('/', 1)[1]):
-            fitem = open(item, 'r')
+            fitem = io.open(item, encoding='utf-8')
             itemtext = fitem.read()
             fitem.close()
             # only the basename of the file
@@ -223,12 +228,14 @@ def write_helpindex(helpdir):
     indexstring = s.substitute(indexbody=htmlstring, css=csstempl,
                                footer=footertempl)
 
-    f_helpindex = open(os.path.join(helpdir, 'helpindex.html'), 'w')
+    f_helpindex = io.open(os.path.join(helpdir, 'helpindex.html'), mode='w',
+                          encoding='utf-8')
     f_helpindex.write(indexstring)
     f_helpindex.close()
 
     # Todo: using string template for .hlp
-    f_helphlpindex = open(os.path.join(helpdir, 'helpindex.hlp'), 'w')
+    f_helphlpindex = io.open(os.path.join(helpdir, 'helpindex.hlp'), mode='w',
+                             encoding='utf-8')
     f_helphlpindex.write('\n'.join(hlp_list))
     f_helphlpindex.close()