Always specify encoding when opening text files

webware/Admin/View.py

@@ -24,8 +24,7 @@ def defaultAction(self):
                 self.writeError('File cannot be viewed!')
 
     def writeError(self, message):
-        self.writeln(
-            f'<h3 style="color:red">Error</h3><p>{message}</p>')
+        self.writeln(f'<h3 style="color:red">Error</h3><p>{message}</p>')
 
     def writeContent(self):
         filename = self.request().field('filename', None)
@@ -41,7 +40,7 @@ def writeContent(self):
         self._type = 'text/' + (
             'html' if splitext(filename)[1] in ('.htm', '.html') else 'plain')
         try:
-            with open(filename) as f:
+            with open(filename, encoding='utf-8') as f:
                 self._data = f.read()
         except Exception:
             self.writeError(f'The requested file {filename!r} cannot be read.')

webware/Application.py

@@ -221,7 +221,8 @@ def __init__(self, path=None, settings=None, development=None):
             if '/' not in filename:
                 filename = os.path.join(self._logDir, filename)
             # pylint: disable=consider-using-with
-            sys.stderr = sys.stdout = open(filename, 'a', buffering=1)
+            sys.stderr = sys.stdout = open(
+                filename, 'a', encoding='utf-8', buffering=1)
 
         self.initErrorPage()
         self.printStartUpMessage()
@@ -267,7 +268,7 @@ def initErrorPage(self):
                      os.path.dirname(os.path.abspath(__file__))):
             error404file = os.path.join(path, 'error404.html')
             try:
-                with open(error404file) as f:
+                with open(error404file, encoding='utf-8') as f:
                     self._error404 = f.read()
             except Exception:
                 continue
@@ -611,7 +612,7 @@ def writeActivityLog(self, trans):
             filename = os.path.join(self._logDir, filename)
         filename = self.serverSidePath(filename)
         mode = 'a' if os.path.exists(filename) else 'w'
-        with open(filename, mode) as f:
+        with open(filename, mode, encoding='utf-8') as f:
             if mode == 'w':
                 f.write(','.join(self.setting('ActivityLogColumns')) + '\n')
             values = []

webware/Examples/Colorize.py

@@ -54,7 +54,7 @@ def respond(self, transaction):
         filename = req.field('filename')
         filename = self.request().serverSidePath(os.path.basename(filename))
         try:
-            with open(filename) as f:
+            with open(filename, encoding='utf-8') as f:
                 source = f.read()
         except IOError:
             source = None

webware/Examples/DBUtilsDemo.py

@@ -44,9 +44,8 @@ class DBUtilsDemo(ExamplePage):
     # Initialize the buttons
     _actions = []
     _buttons = []
-    for action in (
-            'create tables', 'list seminars', 'list attendees',
-            'add seminar', 'add attendee'):
+    for action in ('create tables', 'list seminars', 'list attendees',
+                   'add seminar', 'add attendee'):
         value = action.capitalize()
         action = action.split()
         action[1] = action[1].capitalize()

webware/ExceptionHandler.py

@@ -421,7 +421,7 @@ def saveErrorPage(self, html):
         filename = os.path.join(
             self._app._errorMessagesDir, self.errorPageFilename())
         try:
-            with open(filename, 'w') as f:
+            with open(filename, 'w', encoding='utf-8') as f:
                 f.write(html)
         except IOError:
             sys.stderr.write(
@@ -473,10 +473,10 @@ def fixElement(element):
         filename = self._app.setting('ErrorLogFilename')
         try:
             if os.path.exists(filename):
-                with open(filename, 'a') as f:
+                with open(filename, 'a', encoding='utf-8') as f:
                     f.write(logLine)
             else:
-                with open(filename, 'w') as f:
+                with open(filename, 'w', encoding='utf-8') as f:
                     f.write(
                         'time,filename,pathname,exception name,'
                         'exception data,error report filename\n')

webware/HTTPRequest.py

@@ -71,7 +71,7 @@ def __init__(self, requestDict=None):
 
         # Debugging
         if debug:
-            with open('env.text', 'a') as f:
+            with open('env.text', 'a', encoding='utf-8') as f:
                 f.write('>> env for request:\n')
                 for key in sorted(env):
                     f.write(f'{key}: {env[key]!r}\n')

webware/MiscUtils/DataTable.py

@@ -507,8 +507,8 @@ def strip(x):
     def save(self):
         self.writeFileNamed(self._filename)
 
-    def writeFileNamed(self, filename):
-        with open(filename, 'w') as f:
+    def writeFileNamed(self, filename, encoding='utf-8'):
+        with open(filename, 'w', encoding=encoding) as f:
             self.writeFile(f)
 
     def writeFile(self, file):

webware/MiscUtils/PickleCache.py

@@ -92,7 +92,7 @@ def read(self, filename,
         if not os.path.exists(filename):
             if v:
                 print(f'Cannot find {filename!r}.')
-            open(filename)  # to get a properly constructed IOError
+            open(filename, 'rb')  # to get a properly constructed IOError
 
         shouldDeletePickle = False
         data = None

webware/MiscUtils/PropertiesObject.py

@@ -71,7 +71,7 @@ def loadValues(self, *args, **kwargs):
 
     def readFileNamed(self, filename):
         results = {}
-        with open(filename) as f:
+        with open(filename, encoding='utf-8') as f:
             exec(f.read(), results)
         self.update(results)
         self.cleanPrivateItems()

webware/MiscUtils/Tests/BenchCSVParser.py

@@ -44,8 +44,8 @@ def _main(self):
             print("Benchmark using", name, "...")
             self.benchFileNamed(name)
 
-    def benchFileNamed(self, name):
-        with open(name) as f:
+    def benchFileNamed(self, name, encoding='utf-8'):
+        with open(name, encoding=encoding) as f:
             lines = f.readlines()
         for line in lines:
             for _iteration in range(self._iterations):

webware/MiscUtils/Tests/BenchDataTable.py

@@ -45,8 +45,8 @@ def _main(self):
             print("Benchmark using", name, "...")
             self.benchFileNamed(name)
 
-    def benchFileNamed(self, name):
-        with open(name) as f:
+    def benchFileNamed(self, name, encoding='utf-8'):
+        with open(name, encoding=encoding) as f:
             contents = f.read()
         for _iteration in range(self._iterations):
             # we duplicate lines to reduce the overhead of the loop

webware/MiscUtils/Tests/TestPickleCache.py

@@ -70,7 +70,7 @@ def testOneIteration(self):
             self.remove(picklePath)
 
     def writeSource(self):
-        with open(self._sourcePath, 'w') as f:
+        with open(self._sourcePath, 'w', encoding='ascii') as f:
             f.write(str(self._data))
 
     def writePickle(self):

webware/PSP/Examples/View.py

@@ -26,7 +26,7 @@ def writeContent(self):
                 self.write('<p style="color:red">'
                            f'No such file {basename} exists</p>')
                 return
-            text = open(filename).read()
+            text = open(filename, encoding='utf-8').read()
             text = self.htmlEncode(text)
             text = text.replace('\n', '<br>').replace('\t', ' '*4)
             self.write(f'<pre>{text}</pre>')

webware/PSP/Generators.py

@@ -303,7 +303,7 @@ def generate(self, writer, phase=None):
         # Cut down on those by using a single res.write on the whole
         # file, after escaping any triple-double quotes.
         if self.static:
-            with open(self.page) as f:
+            with open(self.page, encoding='utf-8') as f:
                 data = f.read()
             data = data.replace('"""', r'\"""')
             writer.println(f'res.write("""{data}""")')

webware/PSP/ServletWriter.py

@@ -63,7 +63,7 @@ def close(self):
         self._file.close()
         if os.path.exists(self._pyFilename):
             os.remove(self._pyFilename)
-        with open(self._pyFilename, 'w') as f:
+        with open(self._pyFilename, 'w', encoding='utf-8') as f:
             f.write(pyCode)
 
     def pushIndent(self):

webware/PSP/StreamReader.py

@@ -91,7 +91,7 @@ def pushFile(self, filepath, encoding=None):
         if parent is not None and not isAbsolute:
             filepath = os.path.join(parent, filepath)
         fileId = self.registerSourceFile(filepath)
-        with open(filepath, 'r') as handle:
+        with open(filepath, 'r', encoding=encoding) as handle:
             stream = handle.read()
             handle.seek(0, 0)
             if self.current is None:

webware/PSP/Tests/TestCompiler.py

@@ -50,7 +50,7 @@ def compileString(self, pspSource, classname):
         modulePath = os.path.join(self.testDir, moduleName)
         tmpInName = modulePath + '.psp'
         tmpOutName = modulePath + '.py'
-        with open(tmpInName, 'w') as fp:
+        with open(tmpInName, 'w', encoding='utf-8') as fp:
             fp.write(pspSource)
         # Compile PSP into .py file
         context = Context.PSPCLContext(tmpInName)

webware/Scripts/MakeAppWorkDir.py

@@ -116,7 +116,8 @@ def makeDirectories(self):
                 self.msg(f"\t{path} already exists.")
             else:
                 os.makedirs(path)
-                open(os.path.join(path, '__init__.py'), 'w').write('#\n')
+                open(os.path.join(path, '__init__.py'), 'w',
+                     encoding='ascii').write('#\n')
                 self.msg(f"\t{path} created.")
 
     def copyConfigFiles(self):
@@ -157,14 +158,14 @@ def setLibDirs(self):
         self.msg("Setting the library directories...")
         wsgiScript = os.path.join(self._workDir, 'Scripts', 'WSGIScript.py')
         if os.path.isfile(wsgiScript):
-            with open(wsgiScript) as f:
+            with open(wsgiScript, encoding='utf-8') as f:
                 script = f.read()
             if 'libDirs = []' not in script:
                 self.msg("\tWarning: Unexpected WSGI script")
             else:
                 script = script.replace(
                     'libDirs = []', f'libDirs = {self._libraryDirs!r}')
-                with open(wsgiScript, 'w') as f:
+                with open(wsgiScript, 'w', encoding='utf-8') as f:
                     f.write(script)
         else:
             self.msg("\tWarning: Cannot find WSGI script.")
@@ -192,14 +193,14 @@ def makeDefaultContext(self):
                 self.msg(f"\t{filename} already exists.")
             else:
                 self.msg(f"\t{filename}")
-                open(filename, "w").write(exampleContext[name])
+                open(filename, "w",
+                     encoding='ascii').write(exampleContext[name])
         self.msg("Updating config for default context...")
-        filename = os.path.join(
-            self._workDir, 'Configs', 'Application.config')
+        filename = os.path.join(self._workDir, 'Configs', 'Application.config')
         self.msg(f"\t{filename}")
-        content = open(filename).readlines()
+        content = open(filename, encoding='utf-8').readlines()
         foundContext = 0
-        with open(filename, 'w') as output:
+        with open(filename, 'w', encoding='utf-8') as output:
             for line in content:
                 contextName = self._contextName
                 if line.startswith(
@@ -226,7 +227,7 @@ def addGitIgnore(self):
         if os.path.exists(filename):
             existed = True
         else:
-            with open(filename, 'w') as f:
+            with open(filename, 'w', encoding='ascii') as f:
                 f.write(ignore)
         if existed:
             self.msg("\tDid not change existing .gitignore file.")

webware/Scripts/WaitressServer.py

@@ -54,7 +54,7 @@ def openBrowser():
         del environ['WEBWARE_DEVELOPMENT']
     try:
         # get application from WSGI script
-        with open(args.wsgi_script) as f:
+        with open(args.wsgi_script, encoding='utf-8') as f:
             script = f.read()
         # set development flag in the script
         script = script.replace(

webware/ServletFactory.py

@@ -189,7 +189,7 @@ def _importModuleFromDirectory(
                 # if it does not exist, make an empty one
                 print(f"Creating __init__.py file at {packageDir!r}")
                 try:
-                    with open(initPy, 'w') as initPyFile:
+                    with open(initPy, 'w', encoding='ascii') as initPyFile:
                         initPyFile.write(
                             '# Auto-generated by Webware' + os.linesep)
                 except Exception:

webware/Testing/Main.py

@@ -54,7 +54,7 @@ def readFileNamed(self, filename):
         Each of them is a dictionary, as defined the given file.
         See TestCases.data for information on the format.
         """
-        with open(filename) as f:
+        with open(filename, encoding='utf-8') as f:
             cases = self.readFile(f)
         return cases
 

webware/Tests/TestEndToEnd/TestMakeApp.py

@@ -62,14 +62,14 @@ def testMakeNewApp(self):
         appFiles = '.gitignore error404.html Scripts/WSGIScript.py'.split()
         for appFile in appFiles:
             self.assertTrue(os.path.isfile(appFile), appFile)
-        with open('Scripts/WSGIScript.py') as f:
+        with open('Scripts/WSGIScript.py', encoding='utf-8') as f:
             wsgiScript = f.read().splitlines()
         self.assertEqual(wsgiScript[0], '#!/usr/bin/env python3')
         self.assertIn('from Application import Application', wsgiScript)
         os.chdir('MyContext')
-        with open('__init__.py') as f:
+        with open('__init__.py', encoding='utf-8') as f:
             initScript = f.read().splitlines()
         self.assertIn('def contextInitialize(application, path):', initScript)
-        with open('Main.py') as f:
+        with open('Main.py', encoding='utf-8') as f:
             mainServlet = f.read().splitlines()
         self.assertIn('class Main(Page):', mainServlet)

webware/URLParser.py

@@ -197,7 +197,8 @@ def addContext(self, name, path):
                             print("Creating __init__.py file"
                                   f" for context {name!r}")
                             try:
-                                with open(initPy, 'w') as initPyFile:
+                                with open(initPy, 'w',
+                                          encoding='ascii') as initPyFile:
                                     initPyFile.write(
                                         '# Auto-generated by Webware' +
                                         os.linesep)

webware/WebUtils/HTMLTag.py

@@ -395,13 +395,13 @@ def __init__(
 
     # region Reading
 
-    def readFileNamed(self, filename, retainRootTag=True):
+    def readFileNamed(self, filename, retainRootTag=True, encoding='utf-8'):
         """Read the given file.
 
         Relies on readString(). See that method for more information.
         """
         self._filename = filename
-        with open(filename) as f:
+        with open(filename, encoding=encoding) as f:
             contents = f.read()
         return self.readString(contents, retainRootTag)