Update Standards to Python 3.9 (#292)

regex strings should be raw strings with correct escape sequences
Raw strings should not be raised as errors, ValueError is the default
Added config item for default file encoding, set to utf-8. Previously we were blindly using the system encoding with unpredictable results
Forced all filesystem read and write activities to actively select encoding per Python standards
Excluded warning about use of new python f-strings, as they are not backwards compatible
Added Python 3.9 for test support
Added Test support for Amazon Linux 2 and better filesystem error recognition across Python versions

Signed-off-by: Daniel Chaffelson <chaffelson@gmail.com>

nipyapi/canvas.py

@@ -590,8 +590,8 @@ def delete_processor(processor, refresh=True, force=False):
         target = processor
     if force:
         if not schedule_processor(target, False):
-            raise ("Could not prepare processor {0} for deletion"
-                   .format(target.id))
+            raise ValueError("Could not prepare processor {0} for deletion"
+                             .format(target.id))
         inbound_cons = [
             x for x in get_component_connections(processor)
             if processor.id == x.destination_id

nipyapi/config.py

@@ -152,6 +152,9 @@
     nifi_config.cert_file = os.getenv('NIFI_CLIENT_CERT')
     nifi_config.key_file = os.getenv('NIFI_CLIENT_KEY')
 
-# --- URL Encoding
+# --- Encoding
 # URL Encoding bypass characters will not be encoded during submission
 default_safe_chars = ''
+
+# Default String Encoding
+default_string_encoding = 'utf8'

nipyapi/nifi/api_client.py

@@ -256,15 +256,15 @@ def __deserialize(self, data, klass):
 
         if type(klass) == str:
             if klass.startswith('list['):
-                sub_kls = re.match('list\[(.*)\]', klass).group(1)
+                sub_kls = re.match(r'list\[(.*)]', klass).group(1)
                 if isinstance(data, dict):
                     # ok, we got a single instance when we may have gotten a list
                     return self.__deserialize(data, sub_kls)
                 return [self.__deserialize(sub_data, sub_kls)
                         for sub_data in data]
 
             if klass.startswith('dict('):
-                sub_kls = re.match('dict\(([^,]*), (.*)\)', klass).group(2)
+                sub_kls = re.match(r'dict\(([^,]*), (.*)\)', klass).group(2)
                 return {k: self.__deserialize(v, sub_kls)
                         for k, v in iteritems(data)}
 

nipyapi/parameters.py

@@ -89,7 +89,7 @@ def create_parameter_context(name, description=None, parameters=None):
             component=ParameterContextDTO(
                 name=name,
                 description=description,
-                parameters=parameters if parameters else list()
+                parameters=parameters if parameters else []
                 # list() per NiFi Jira 7995
             )
         )

nipyapi/templates.py

@@ -7,6 +7,7 @@
 from __future__ import absolute_import
 
 import json
+import io
 from os import access, R_OK, W_OK
 from os.path import isfile, dirname
 import logging
@@ -283,7 +284,7 @@ def load_template_from_xml_file_path(file_path):
     """
     assert isfile(file_path) and access(file_path, R_OK), \
         SystemError("File {0} invalid or unreadable".format(file_path))
-    with open(file_path, "r") as template_file:
+    with io.open(file_path, "r", encoding='utf8') as template_file:
         return load_template_from_xml_file_stream(template_file)
 
 

nipyapi/utils.py

@@ -9,6 +9,7 @@
 import logging
 import json
 import re
+import io
 import time
 from copy import copy
 from functools import reduce, wraps
@@ -23,6 +24,7 @@
 from requests.models import Response
 from future.utils import raise_from as _raise
 import nipyapi
+from nipyapi.config import default_string_encoding as DEF_ENCODING
 
 __all__ = ['dump', 'load', 'fs_read', 'fs_write', 'filter_obj',
            'wait_to_complete', 'is_endpoint_up', 'set_endpoint',
@@ -126,8 +128,12 @@ def fs_write(obj, file_path):
     Returns: The object that was written
     """
     try:
-        with open(str(file_path), 'w') as f:
-            f.write(obj)
+        with io.open(str(file_path), 'w', encoding=DEF_ENCODING) as f:
+            if isinstance(obj, bytes):
+                obj_str = obj.decode(DEF_ENCODING)
+            else:
+                obj_str = obj
+            f.write(obj_str)
         return obj
     except TypeError as e:
         raise e
@@ -143,7 +149,7 @@ def fs_read(file_path):
     Returns: The object that was read
     """
     try:
-        with open(str(file_path), 'r') as f:
+        with io.open(str(file_path), 'r', encoding=DEF_ENCODING) as f:
             return f.read()
     except IOError as e:
         raise e
@@ -527,7 +533,7 @@ def validate_parameters_versioning_support():
     if nifi_check or registry_check:
         log.warning("Connected NiFi Registry may not support "
                     "Parameter Contexts and they may be lost in "
-                    "Version Control".format())
+                    "Version Control")
 
 
 def enforce_min_ver(min_version, bool_response=False, service='nifi'):

pylintrc

@@ -50,7 +50,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call, C0103, E1101, R1710, E0401
+disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call, C0103, E1101, R1710, E0401, C0209
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

tests/test_utils.py

@@ -4,18 +4,20 @@
 """Tests for `nipyapi` _utils package."""
 
 from __future__ import absolute_import
+import sys
 import pytest
 import six
 from tests import conftest
 import json
 from deepdiff import DeepDiff
 from nipyapi import utils, nifi, system
+from nipyapi.config import default_string_encoding as DEF_ENCODING
 
 
 def test_dump(regress_flow_reg, fix_flow_serde):
     # Testing that we don't modify or lose information in the round trip
     # Processing in memory for json
-    export_obj = json.loads(fix_flow_serde.raw.decode('utf-8'))
+    export_obj = json.loads(fix_flow_serde.raw.decode(DEF_ENCODING))
     ss_json = utils.dump(
         obj=export_obj,
         mode='json'
@@ -68,13 +70,20 @@ def test_fs_write(tmpdir):
     )
     assert r1 == test_obj
     # Test writing to an invalid location
-    with pytest.raises(IOError):
-        _ = utils.fs_write(
-            obj=test_obj,
-            file_path='/dev/AlmostCertainlyNotAValidDevice'
-        )
+    if sys.version_info >= (3,3):
+        with pytest.raises((OSError, IOError, PermissionError)):
+            _ = utils.fs_write(
+                obj=test_obj,
+                file_path='/dev/AlmostCertainlyNotAValidDevice'
+            )
+    else:
+        with pytest.raises((OSError, IOError)):
+            _ = utils.fs_write(
+                obj=test_obj,
+                file_path='/dev/AlmostCertainlyNotAValidDevice'
+            )
     # Test writing an invalid object
-    with pytest.raises(TypeError):
+    with pytest.raises((TypeError,AttributeError)):
         _ = utils.fs_write(
             obj={},
             file_path=f_fpath
@@ -87,10 +96,16 @@ def test_fs_read(fix_flow_serde):
     )
     assert r1 == fix_flow_serde.json
     # Test reading from unreachable file
-    with pytest.raises(IOError):
-        _ = utils.fs_read(
-            file_path='/dev/AlmostCertainlyNotAValidDevice'
-        )
+    if sys.version_info >= (3,3):
+        with pytest.raises((OSError, IOError, FileNotFoundError, PermissionError)):
+            _ = utils.fs_read(
+                file_path='/dev/AlmostCertainlyNotAValidDevice'
+            )
+    else:
+        with pytest.raises((OSError, IOError)):
+            _ = utils.fs_read(
+                file_path='/dev/AlmostCertainlyNotAValidDevice'
+            )
 
 
 def test_filter_obj(fix_pg):

tox.ini

@@ -3,7 +3,7 @@ envlist = py27, py3, flake8, lint
 
 [travis]
 python =
-    3.8: py3
+    3.9: py3
     2.7: py27
 
 [flake8]