Ensure all files written by AiiDA and plugins are utf8 encoded (#2107)

* Changed all open for `io.open` and specified UTF8 encoding
* Convert explicitly set byte strings to unicode when writing
* `json.dumps` doesn't always produce a unicode string (e.g. Python 2 with a bytestring
  as input), therefore we use `six.text_type` to convert it to unicode and pass it to
  `io.open(demofile.txt, w, encoding='utf-8')` which expects a unicode object and then
  it encodes it utf-8.
* Set literal strings as unicode
* Add encoding option to YAML to ensure that `yaml.dump` returns a bytes object in both Py2 and 3
* Add `json.py` helper function to replace system json
* Replace json imports with `from aiida.utils import json`

.pre-commit-config.yaml

@@ -196,11 +196,9 @@
         aiida/common/profile.py|
         aiida/common/setup.py|
         aiida/common/test_extendeddicts.py|
-        aiida/common/test_folders.py|
         aiida/common/test_logging.py|
         aiida/common/test_utils.py|
         aiida/control/tests/test_postgres.py|
-        aiida/daemon/client.py|
         aiida/daemon/execmanager.py|
         aiida/daemon/runner.py|
         aiida/daemon/timestamps.py|

aiida/backends/djsite/cmdline.py

@@ -13,7 +13,7 @@
 from __future__ import print_function
 from __future__ import absolute_import
 import datetime
-import json
+import aiida.utils.json as json
 
 from django.db.models import Q
 

aiida/backends/djsite/db/models.py

@@ -333,9 +333,9 @@ def _deserialize_attribute(mainitem, subitems, sep, original_class=None,
     :return: the deserialized value
     :raise aiida.backends.djsite.db.models.DeserializationException: if an error occurs
     """
+    import aiida.utils.json as json
     from aiida.utils.timezone import (
         is_naive, make_aware, get_current_timezone)
-    import json
 
     from aiida.common import aiidalogger
 
@@ -729,8 +729,9 @@ def create_value(cls, key, value, subspecifier_value=None,
           responsibility to store such entries (typically with a Django
           bulk_create() call).
         """
-        import json
         import datetime
+
+        import aiida.utils.json as json
         from aiida.utils.timezone import is_naive, make_aware, get_current_timezone
 
         if cls._subspecifier_field_name is None:
@@ -1402,7 +1403,7 @@ def get_aiida_class(self):
         return DjangoComputer.from_dbmodel(self, construct_backend())
 
     def _get_val_from_metadata(self, key):
-        import json
+        import aiida.utils.json as json
 
         try:
             metadata = json.loads(self.metadata)
@@ -1680,7 +1681,7 @@ class Meta:
     def set_value(self, arg):
         from aiida.orm.node import Node
         from aiida.common.datastructures import wf_data_value_types
-        import json
+        import aiida.utils.json as json
 
         try:
             if isinstance(arg, Node) or issubclass(arg.__class__, Node):
@@ -1697,7 +1698,8 @@ def set_value(self, arg):
             six.reraise(ValueError, "Cannot set the parameter {}".format(self.name), sys.exc_info()[2])
 
     def get_value(self):
-        import json
+        import aiida.utils.json as json
+
         from aiida.common.datastructures import wf_data_value_types
 
         if self.value_type == wf_data_value_types.JSON:

aiida/backends/djsite/utils.py

@@ -61,7 +61,7 @@ def get_log_messages(obj):
         which the log message was issued, as well as additional 'metadata'
     """
     from aiida.backends.djsite.db.models import DbLog
-    import json
+    import aiida.utils.json as json
 
     extra = get_dblogger_extra(obj)
 

aiida/backends/sqlalchemy/models/authinfo.py

@@ -11,7 +11,6 @@
 from __future__ import division
 from __future__ import print_function
 from __future__ import absolute_import
-import json
 
 from sqlalchemy import ForeignKey
 from sqlalchemy.orm import relationship

aiida/backends/sqlalchemy/models/computer.py

@@ -11,7 +11,6 @@
 from __future__ import division
 from __future__ import print_function
 from __future__ import absolute_import
-import json
 
 import six
 

aiida/backends/sqlalchemy/models/workflow.py

@@ -11,7 +11,6 @@
 from __future__ import division
 from __future__ import print_function
 from __future__ import absolute_import
-import json
 
 from sqlalchemy import ForeignKey
 from sqlalchemy.orm import relationship, backref
@@ -27,6 +26,7 @@
 from aiida.common.datastructures import (wf_states, wf_data_types,
                                          wf_data_value_types, wf_default_call)
 from aiida.utils import timezone
+import aiida.utils.json as json
 
 
 

aiida/backends/sqlalchemy/utils.py

@@ -21,7 +21,7 @@
     json_dumps = partial(json.dumps, double_precision=15)
     json_loads = partial(json.loads, precise_float=True)
 except ImportError:
-    import json
+    import aiida.utils.json as json
 
     json_dumps = json.dumps
     json_loads = json.loads

aiida/backends/tests/backup_script.py

@@ -12,19 +12,19 @@
 from __future__ import absolute_import
 import datetime
 import importlib
-import json
 import shutil
 import sys
 import tempfile
 
 from dateutil.parser import parse
 
-from aiida.common import utils
-from aiida.common.additions.backup_script import backup_setup
-from aiida.orm.node import Node
 from aiida.backends.utils import is_dbenv_loaded, load_dbenv, BACKEND_SQLA, BACKEND_DJANGO
 from aiida.backends.settings import BACKEND
 from aiida.backends.testbase import AiidaTestCase
+from aiida.common import utils
+from aiida.common.additions.backup_script import backup_setup
+from aiida.orm.node import Node
+import aiida.utils.json as json
 
 
 if not is_dbenv_loaded():

aiida/backends/tests/backup_setup_script.py

@@ -11,7 +11,7 @@
 from __future__ import division
 from __future__ import print_function
 from __future__ import absolute_import
-import json
+import io
 import shutil
 import tempfile
 import os
@@ -20,7 +20,7 @@
 from aiida.common.additions.backup_script import backup_setup
 from aiida.common.additions.backup_script.backup_base import AbstractBackup
 from aiida.backends.testbase import AiidaTestCase
-
+import aiida.utils.json as json
 
 
 class TestBackupSetupScriptUnit(AiidaTestCase):
@@ -120,7 +120,7 @@ def test_full_backup_setup_script(self):
                             "It contains: {}.".format(backup_conf_records))
 
             # Check the content of the main backup configuration file
-            with open(os.path.join(temp_aiida_folder, "backup_info.json")
+            with io.open(os.path.join(temp_aiida_folder, "backup_info.json"), encoding='utf8'
                       ) as conf_jfile:
                 conf_cont = json.load(conf_jfile)
                 self.assertEqual(conf_cont[AbstractBackup.OLDEST_OBJECT_BK_KEY],