Implement safe_open() to open text files in utf-8

- Should fix #505
- Pending testing

libopensesame/experiment.py

@@ -564,16 +564,16 @@ def save(self, path, overwrite=False, update_path=True):
 		# If there are no files in the pool, save the script as plain text
 		if self.pool.count_included() == 0:
 			debug.msg(u'saving as plain text (without file pool)')
-			with open(path, u'w') as fd:
-				fd.write(safe_str(self.to_string()))
+			with safe_open(path, u'w') as fd:
+				fd.write(self.to_string())
 			self.experiment_path = os.path.dirname(path)
 			return path
 		debug.msg(u'saving as .tar.gz archive (with file pool)')
 		# Write the script to a text file
 		script = self.to_string()
 		script_path = os.path.join(self.pool.folder(), u'script.opensesame')
-		with open(script_path, u'w') as fd:
-			fd.write(safe_str(script))
+		with safe_open(script_path, u'w') as fd:
+			fd.write(script)
 		# Create the archive in a a temporary folder and move it afterwards.
 		# This hack is needed, because tarfile fails on a Unicode path.
 		tmp_path = tempfile.mktemp(suffix=u'.osexp')
@@ -638,8 +638,8 @@ def open(self, src):
 		except tarfile.ReadError:
 			# If the file wasn't a .tar.gz, then it must be a plain-text file
 			debug.msg(u"opening plain-text experiment")
-			with open(src, universal_newline_mode) as fd:
-				return safe_decode(fd.read())
+			with safe_open(src, universal_newline_mode) as fd:
+				return fd.read()
 		debug.msg(u"opening .tar.gz archive")
 		# If the file is a .tar.gz archive, extract the pool to the pool folder
 		# and return the contents of opensesame.script.
@@ -667,8 +667,8 @@ def open(self, src):
 				os.rmdir(os.path.join(self.pool.folder(), folder))
 		script_path = os.path.join(self.pool.folder(), u"script.opensesame")
 		tar.extract(u"script.opensesame", self.pool.folder())
-		with open(script_path, universal_newline_mode) as fd:
-			script = safe_decode(fd.read())
+		with safe_open(script_path, universal_newline_mode) as fd:
+			script = fd.read()
 		os.remove(script_path)
 		self.experiment_path = os.path.dirname(src)
 		return script

libopensesame/misc.py

@@ -39,7 +39,7 @@ def parse_environment_file():
 	if not os.path.exists(u'environment.yaml'):
 		return
 	import yaml
-	with open(u'environment.yaml') as fd:
+	with safe_open(u'environment.yaml') as fd:
 		d = yaml.load(fd.read())
 	# Convert all values from UTF8 to the filesystem encoding
 	for key, val in d.items():

libopensesame/py3compat.py

@@ -17,7 +17,9 @@
 along with OpenSesame.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+import functools
 import sys
+import io
 
 if sys.version_info >= (3,0,0):
 	py3 = True
@@ -63,14 +65,15 @@ def safe_read(path):
 	with open(path, u'r') as fd:
 		return safe_decode(fd.read(), errors=u'ignore')
 
+safe_open = functools.partial(io.open, encoding=u'utf-8')
 
 if py3:
 	safe_str = safe_decode
 else:
 	safe_str = safe_encode
 
 __all__ = ['py3', 'safe_decode', 'safe_encode', 'safe_str',
-	'universal_newline_mode', 'safe_read']
+	'universal_newline_mode', 'safe_read', 'safe_open']
 if not py3:
 	__all__ += ['str', 'bytes']
 else:

libopensesame/python_workspace.py

@@ -119,7 +119,7 @@ def run_file(self, path):
 				type:	str
 		"""
 
-		with open(path) as fd:
+		with safe_open(path) as fd:
 			script = fd.read()
 		bytecode = self._compile(script)
 		self._exec(bytecode)

libqtopensesame/misc/base_component.py

@@ -88,12 +88,8 @@ def load_ui(self, ui=None):
 					from libopensesame import misc
 					ui_path = misc.resource(os.path.join(*path_list)+u'.ui')
 			debug.msg(u'dynamically loading ui: %s' % ui_path)
-			if py3:
-				with open(ui_path, encoding=u'utf-8') as fd:
-					self.ui = uic.loadUi(fd, self)
-			else:
-				with open(ui_path) as fd:
-					self.ui = uic.loadUi(fd, self)
+			with safe_open(ui_path) as fd:
+				self.ui = uic.loadUi(fd, self)
 		else:
 			self.ui = None
 

libqtopensesame/misc/markdown_parser.py

@@ -72,8 +72,8 @@ def __init__(self, main_window):
 
 		self.setup(main_window)
 		self.css = u'<style type="text/css">'
-		with open(self.main_window.theme.resource(u'markdown.css')) as fd:
-			self.css += safe_decode(fd.read()) % {u'background_image' : \
+		with safe_open(self.main_window.theme.resource(u'markdown.css')) as fd:
+			self.css += fd.read() % {u'background_image' : \
 				os.path.abspath(self.main_window.theme.resource(
 				u'background.png'))}
 		if highlight is not None:

libqtopensesame/misc/theme.py

@@ -69,7 +69,7 @@ def __init__(self, main_window, theme=None):
 		if os.path.exists(self.theme_info):
 			info = imp.load_source(self.theme,
 				safe_str(self.theme_info, enc=misc.filesystem_encoding()))
-			with open(os.path.join(self.theme_folder, info.qss)) as fd:
+			with safe_open(os.path.join(self.theme_folder, info.qss)) as fd:
 				self._qss = fd.read()
 			self._icon_map = info.icon_map
 			self._icon_theme = info.icon_theme
@@ -236,9 +236,9 @@ def load_icon_map(self):
 		self.icon_map = {}
 		path = os.path.join(self.theme_folder, self._icon_map)
 		debug.msg(path)
-		with open(path) as fd:
+		with safe_open(path) as fd:
 			for l in fd:
-				l = l.split(",")
+				l = l.split(u',')
 				if len(l) == 3:
 					try:
 						size = int(l[2])

libqtopensesame/qtopensesame.py

@@ -215,7 +215,7 @@ def resume_init(self):
 			tmpl = u'default-py3.osexp'
 		else:
 			tmpl = u'default.osexp'
-		with open(misc.resource(os.path.join(u'templates', tmpl)), u'r') as fd:
+		with safe_open(misc.resource(os.path.join(u'templates', tmpl)), u'r') as fd:
 			self.experiment = experiment.experiment(self, u'New experiment',
 				fd.read())
 		self.experiment.build_item_tree()

openexp/_log/csv.py

@@ -19,7 +19,6 @@
 
 from libopensesame.py3compat import *
 from openexp._log.log import log
-import codecs
 import os
 
 class csv(log):
@@ -52,7 +51,7 @@ def open(self, path):
 		else:
 			self._path = path
 		# Open the logfile
-		self._log = codecs.open(self._path, u'w', u'utf-8')
+		self._log = safe_open(self._path, u'w')
 		self._header_written = False
 
 	def write(self, msg, newline=True):

openexp/backend.py

@@ -50,7 +50,7 @@ def backend_info(experiment):
 	# Use caching
 	if _backend_info is not None:
 		return _backend_info
-	with open(experiment.resource(u'backend_info.yaml')) as fd:
+	with safe_open(experiment.resource(u'backend_info.yaml')) as fd:
 		d = yaml.load(fd)
 	l = []
 	for name, info in d.items():

opensesame_extensions/get_started/get_started.py

@@ -71,8 +71,8 @@ def activate(self):
 					(i, cls, os.path.basename(path))
 				recent.append(md)
 		# Create markdown
-		with open(self.ext_resource(u'get_started.md')) as fd:
-			md = safe_decode(fd.read())
+		with safe_open(self.ext_resource(u'get_started.md')) as fd:
+			md = fd.read()
 		md = md % {
 			u'version' : metadata.__version__,
 			u'codename' : metadata.codename,

opensesame_extensions/help/help.py

@@ -118,7 +118,7 @@ def online_help_menu(self, sitemap_url, base_url, label,
 		except:
 			if local_sitemap is None:
 				return
-			with open(self.ext_resource(local_sitemap)) as fd:
+			with safe_open(self.ext_resource(local_sitemap)) as fd:
 				sitemap = fd.read()
 				_dict = yaml.load(sitemap)
 		if not isinstance(_dict, dict):
@@ -169,7 +169,7 @@ def psychopy_help_menu(self):
 		"""
 
 		import yaml
-		with open(self.ext_resource(u'psychopy_sitemap.yaml')) as fd:
+		with safe_open(self.ext_resource(u'psychopy_sitemap.yaml')) as fd:
 			sitemap = fd.read()
 		_dict = yaml.load(sitemap)
 		menu = self.build_menu(self.menu, None, _(u'PsychoPy API'), _dict)