NiFi 1.15 requires a password of at least 12chars, updating to supers…

…ecret1! as an example

Added separate option to config to disable_insecure_request_warnings in urllib3
Updated service_login to correctly check for None explicitly instead of implicit
Updated set_endpoint to also accept username and password for login behavior
Updated docker-compose configurations for NiFi-1.15, this includes setting default single user credential and port updates for https
Updated conftest to explicitly set login/ssl/username/password from tuples per test permutation rather than implicitly pulling them from config.py defaults
Updated conftest service cleanup actions to be more concise and readable
Corrected test for updating a parameter context to not duplicate keys which is now a testable fault
Updated test_security.py to correctly have the manual testing helper commented out by default

Signed-off-by: Daniel Chaffelson <chaffelson@gmail.com>

nipyapi/config.py

@@ -130,10 +130,11 @@
 # unnecessary warnings
 # Set to True by default, change to false if necessary
 global_ssl_verify = True
+disable_insecure_request_warnings = False
 
 nifi_config.verify_ssl = global_ssl_verify
 registry_config.verify_ssl = global_ssl_verify
-if not global_ssl_verify:
+if not global_ssl_verify or disable_insecure_request_warnings:
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Enforce no host checking when SSL context is disabled

nipyapi/demo/secure_connection.py

@@ -220,7 +220,7 @@ def bootstrap_nifi_access_policies(user='nobel'):
     test_function=nipyapi.security.service_login,
     service='nifi',
     username='nobel',
-    password='password',
+    password='supersecret1!',
     bool_response=True,
     nipyapi_delay=nipyapi.config.long_retry_delay,
     nipyapi_max_wait=nipyapi.config.long_max_wait

nipyapi/security.py

@@ -320,8 +320,8 @@ def service_login(service='nifi', username=None, password=None,
     default_pword = getattr(nipyapi.config, 'default_' + service + '_password')
     default_uname = getattr(nipyapi.config, 'default_' + service + '_username')
     # We use copy so we don't clobber the default by mistake
-    pword = password if password else copy(default_pword)
-    uname = username if username else copy(default_uname)
+    pword = password if password is not None else copy(default_pword)
+    uname = username if username is not None else copy(default_uname)
     assert pword, "Password must be set or in default config"
     assert uname, "Username must be set or in default config"
     # set username/password in configuration for initial login

nipyapi/utils.py

@@ -292,7 +292,8 @@ def is_endpoint_up(endpoint_url):
         return False
 
 
-def set_endpoint(endpoint_url, ssl=False, login=False):
+def set_endpoint(endpoint_url, ssl=False, login=False,
+                 username=None, password=None):
     """
     EXPERIMENTAL
 
@@ -306,6 +307,8 @@ def set_endpoint(endpoint_url, ssl=False, login=False):
           nipyapi.config to authenticate if a secure URL is detected
         login (bool): Whether to attempt login using default cred in config
           requires ssl to be set
+        username (str): The username to use for login, if specified
+        password (str): The password to use for login, if specified
 
     Returns (bool): True for success, False for not
     """
@@ -337,7 +340,9 @@ def set_endpoint(endpoint_url, ssl=False, login=False):
                 service=service,
                 ca_file=nipyapi.config.default_ssl_context['ca_file']
             )
-            nipyapi.security.service_login(service)
+            nipyapi.security.service_login(
+                service, username=username, password=password
+            )
     return True
 
 

resources/docker/latest/docker-compose.yml

@@ -1,14 +1,17 @@
-version: '2'
+version: '3'
 # the latest tag is not pulling the latest image on Travis, so setting explicit
 services:
   nifi:
-    image: apache/nifi:1.13.2
+    image: apache/nifi:1.15.0
     container_name: nifi
     hostname: nifi
     ports:
-      - "8080:8080"
+      - "8443:8443"
+    environment:
+      - SINGLE_USER_CREDENTIALS_USERNAME=nobel
+      - SINGLE_USER_CREDENTIALS_PASSWORD=supersecret1!
   registry:
-    image: apache/nifi-registry:0.8.0
+    image: apache/nifi-registry:1.15.0
     container_name: registry
     hostname: registry
     ports:

resources/docker/secure/docker-compose.yml

@@ -5,7 +5,7 @@ services:
     container_name: secure-nifi
     hostname: secure-nifi
     ports:
-      - "8443:8443"
+      - "9443:8443"
     volumes:
       - ../../../nipyapi/demo/keys:/opt/certs:ro  # Min docker version tested 18, does not work on old docker
     environment:
@@ -24,6 +24,7 @@ services:
       - LDAP_USER_SEARCH_FILTER=(uid={0})
       - LDAP_IDENTITY_STRATEGY=USE_USERNAME
       - LDAP_URL=ldap://ldap.forumsys.com:389
+      - NIFI_WEB_PROXY_HOST=localhost:9443,localhost:8443
   secure-registry:
     image: apache/nifi-registry:0.8.0
     container_name: secure-registry

resources/docker/tox-full/docker-compose.yml

@@ -25,11 +25,14 @@ services:
     ports:
       - "10192:8080"
   nifi:
-    image: apache/nifi:1.13.2
+    image: apache/nifi:1.15.0
     container_name: nifi
     hostname: nifi
     ports:
-      - "8080:8080"
+      - "8443:8443"
+    environment:
+      - SINGLE_USER_CREDENTIALS_USERNAME=nobel
+      - SINGLE_USER_CREDENTIALS_PASSWORD=supersecret1!
   registry-010:
     image: apache/nifi-registry:0.1.0
     container_name: registry-010
@@ -47,7 +50,7 @@ services:
     environment:
       - NIFI_REGISTRY_WEB_HTTP_PORT=18030
   registry:
-    image: apache/nifi-registry:0.8.0
+    image: apache/nifi-registry:1.15.0
     container_name: registry
     hostname: registry
     ports:

tests/conftest.py

@@ -9,6 +9,7 @@
 from os import environ, path
 from collections import namedtuple
 from time import sleep
+
 import nipyapi
 
 log = logging.getLogger(__name__)
@@ -53,34 +54,34 @@
 # Can't use skiptest with parametrize for Travis
 # Mostly because loading up all the environments takes too long
 
-default_nifi_endpoints = ['http://' + test_host + ':8080/nifi-api']
+default_nifi_endpoints = [('https://' + test_host + ':8443/nifi-api', True, True, 'nobel', 'supersecret1!')]
 regress_nifi_endpoints = [
-            'http://' + test_host + ':10112/nifi-api',
-            'http://' + test_host + ':10120/nifi-api',
-            'http://' + test_host + ':10180/nifi-api',
-            'http://' + test_host + ':10192/nifi-api',
-        ]
-secure_nifi_endpoints = ['https://' + test_host + ':8443/nifi-api']
+    ('http://' + test_host + ':10112/nifi-api', True, True, None, None),
+    ('http://' + test_host + ':10120/nifi-api', True, True, None, None),
+    ('http://' + test_host + ':10180/nifi-api', True, True, None, None),
+    ('http://' + test_host + ':10192/nifi-api', True, True, None, None),
+]
+secure_nifi_endpoints = [('https://' + test_host + ':9443/nifi-api', True, True, 'nobel', 'password')]
 default_registry_endpoints = [
-        ('http://' + test_host + ':18080/nifi-registry-api',
-         'http://registry:18080',
-         'http://' + test_host + ':8080/nifi-api'
-         )
-    ]
+    (('http://' + test_host + ':18080/nifi-registry-api', True, True, None, None),
+     'http://registry:18080',
+     ('https://' + test_host + ':8443/nifi-api', True, True, 'nobel', 'supersecret1!')
+     )
+]
 regress_registry_endpoints = [
-            ('http://' + test_host + ':18010/nifi-registry-api',
+            (('http://' + test_host + ':18010/nifi-registry-api', True, True, None, None),
                 'http://registry-010:18010',
-             'http://' + test_host + ':8080/nifi-api'
+             ('https://' + test_host + ':8443/nifi-api', True, True, 'nobel', 'supersecret1!')
              ),
-            ('http://' + test_host + ':18030/nifi-registry-api',
+            (('http://' + test_host + ':18030/nifi-registry-api', True, True, None, None),
                 'http://registry-030:18030',
-             'http://' + test_host + ':10192/nifi-api'
+             ('http://' + test_host + ':10192/nifi-api', True, True, None, None)
              )
         ]
 secure_registry_endpoints = [
-        ('https://' + test_host + ':18443/nifi-registry-api',
+        (('https://' + test_host + ':18443/nifi-registry-api', True, True, None, None),
          'https://secure-registry:18443',
-         'https://' + test_host + ':8443/nifi-api'
+         ('https://' + test_host + ':9443/nifi-api', True, True, 'nobel', 'password')
          )]
 
 if "TRAVIS" in environ and environ["TRAVIS"] == "true":
@@ -99,6 +100,10 @@
     # back each time.
     nifi_test_endpoints = []
     registry_test_endpoints = []
+    if test_default or test_regression:
+        # Added because nifi-1.15+ automatically self-signs certificates for single user mode
+        nipyapi.config.nifi_config.verify_ssl = False
+        nipyapi.config.disable_insecure_request_warnings = True
     if test_default:
         nifi_test_endpoints += default_nifi_endpoints
         registry_test_endpoints += default_registry_endpoints
@@ -140,8 +145,8 @@ def pytest_generate_tests(metafunc):
 @pytest.fixture(scope="function")
 def regress_nifi(request):
     log.info("NiFi Regression test setup called against endpoint %s",
-             request.param)
-    nipyapi.utils.set_endpoint(request.param, True, True)
+             request.param[0])
+    nipyapi.utils.set_endpoint(*request.param)
 
 
 def remove_test_registry_client():
@@ -174,23 +179,23 @@ def ensure_registry_client(uri):
 @pytest.fixture(scope="function")
 def regress_flow_reg(request):
     log.info("NiFi-Registry regression test called against endpoints %s",
-             request.param)
+             request.param[0][0])
     # Set Registry connection
-    nipyapi.utils.set_endpoint(request.param[0], True, True)
+    nipyapi.utils.set_endpoint(*request.param[0])
     # Set paired NiFi connection
-    nipyapi.utils.set_endpoint(request.param[2], True, True)
+    nipyapi.utils.set_endpoint(*request.param[2])
     # because pytest won't let you easily cascade parameters through fixtures
     # we set the docker URI in the config for retrieval later on
     nipyapi.config.registry_local_name = request.param[1]
 
-
 # Tests that the Docker test environment is available before running test suite
 @pytest.fixture(scope="session", autouse=True)
 def session_setup(request):
     log.info("Commencing test session setup")
-    for url in nifi_test_endpoints + [x[0] for x in registry_test_endpoints]:
+    for this_endpoint in nifi_test_endpoints + [x[0] for x in registry_test_endpoints]:
+        url = this_endpoint[0]
         log.debug("Now Checking URL [{0}]".format(url))
-        nipyapi.utils.set_endpoint(url, ssl=True, login=True)
+        nipyapi.utils.set_endpoint(*this_endpoint)
         # ssl and login will only work if https is in the url, else will silently skip
         gui_url = url.replace('-api', '')
         if not nipyapi.utils.wait_to_complete(
@@ -213,7 +218,7 @@ def session_setup(request):
                                  "instead".format(url, api_host))
             log.info("Tested NiFi client connection, got response from %s",
                      url)
-            if 'https://' in url:
+            if url in [x[0] for x in secure_nifi_endpoints]:
                 nipyapi.security.bootstrap_security_policies(service='nifi')
             cleanup_nifi()
         elif 'nifi-registry-api' in url:
@@ -224,8 +229,7 @@ def session_setup(request):
                     nipyapi.security.bootstrap_security_policies(service='registry')
                 cleanup_reg()
             else:
-                raise ValueError("No Response from NiFi-Registry test call"
-                                 )
+                raise ValueError("No Response from NiFi-Registry test call")
         else:
             raise ValueError("Bad API Endpoint")
     request.addfinalizer(final_cleanup)
@@ -283,57 +287,44 @@ def remove_test_buckets():
 
 
 def final_cleanup():
-    for url in nifi_test_endpoints + [x[0] for x in registry_test_endpoints]:
-        nipyapi.utils.set_endpoint(url, True, True)
+    for this_endpoint in nifi_test_endpoints + [x[0] for x in registry_test_endpoints]:
+        url = this_endpoint[0]
+        nipyapi.utils.set_endpoint(*this_endpoint)
         if 'nifi-api' in url:
             cleanup_nifi()
         elif 'nifi-registry-api' in url:
             cleanup_reg()
 
 
 def remove_test_service_users(service='both'):
-    nifi_test_users = [
-        x for x in
-        nipyapi.security.list_service_users('nifi')
-        if x.component.identity.startswith(test_basename)
-    ]
-    reg_test_users = [
-        x for x in
-        nipyapi.security.list_service_users('registry')
-        if x.identity.startswith(test_basename)
-    ]
-    if service != 'registry':
+    if service == 'nifi' or service == 'both':
         _ = [
             nipyapi.security.remove_service_user(x, 'nifi')
-            for x in nifi_test_users
+            for x in
+            nipyapi.security.list_service_users('nifi')
+            if x.component.identity.startswith(test_basename)
         ]
-    if service != 'nifi':
+    if service == 'registry' or service == 'both':
         _ = [
             nipyapi.security.remove_service_user(x, 'registry')
-            for x in reg_test_users
+            for x in
+            nipyapi.security.list_service_users('registry')
+            if x.identity.startswith(test_basename)
         ]
 
 
 def remove_test_service_user_groups(service='both'):
-    nifi_test_user_groups = [
-        x for x in
-        nipyapi.security.list_service_user_groups('nifi')
-        if x.component.identity.startswith(test_basename)
-    ]
-    reg_test_user_groups = [
-        x for x in
-        nipyapi.security.list_service_user_groups('registry')
-        if x.identity.startswith(test_basename)
-    ]
-    if service != 'registry':
+    if service == 'nifi' or service == 'both':
         _ = [
-            nipyapi.security.remove_service_user_group(x, 'nifi')
-            for x in nifi_test_user_groups
+            nipyapi.security.remove_service_user_group(x, 'nifi') for x in
+            nipyapi.security.list_service_user_groups('nifi')
+            if x.component.identity.startswith(test_basename)
         ]
-    if service != 'nifi':
+    if service == 'registry' or service == 'both':
         _ = [
-            nipyapi.security.remove_service_user_group(x, 'registry')
-            for x in reg_test_user_groups
+            nipyapi.security.remove_service_user_group(x, 'registry') for x in
+            nipyapi.security.list_service_user_groups('registry')
+            if x.identity.startswith(test_basename)
         ]
 
 

tests/test_parameters.py

@@ -82,9 +82,9 @@ def test_update_parameter_context(regress_nifi, fix_context):
     )
     assert isinstance(r1, ParameterContextEntity)
     assert 1 == len([x for x in r1.component.parameters if 'Black' in x.parameter.name])
-    r1.component.parameters.append(
+    r1.component.parameters = [
         parameters.prepare_parameter('Black', 'Votes', 'Matter', True)
-    )
+    ]
     r2 = parameters.update_parameter_context(
         r1
     )

tests/test_security.py

@@ -12,10 +12,10 @@
 pytestmark = pytest.mark.skipif(not conftest.test_security, reason='test_security disabled in Conftest')
 
 # Useful for manual testing
-if conftest.test_security:
-    test_host = nipyapi.config.default_host
-    nipyapi.utils.set_endpoint('https://' + test_host + ':18443/nifi-registry-api', True, True)
-    nipyapi.utils.set_endpoint('https://' + test_host + ':8443/nifi-api', True, True)
+# if conftest.test_security:
+#     test_host = nipyapi.config.default_host
+#     nipyapi.utils.set_endpoint('https://' + test_host + ':18443/nifi-registry-api', True, True)
+#     nipyapi.utils.set_endpoint('https://' + test_host + ':9443/nifi-api', True, True)
 
 
 def test_list_service_users():

tests/test_versioning.py

@@ -18,15 +18,15 @@ def test_create_registry_client(regress_flow_reg):
      ]
     r = versioning.create_registry_client(
         name=conftest.test_registry_client_name,
-        uri=conftest.registry_test_endpoints[0][0],
+        uri=conftest.registry_test_endpoints[0][0][0],
         description='a test connection'
     )
     assert isinstance(r, nifi.RegistryClientEntity)
     # test duplicate catch result
     with pytest.raises(ValueError):
         _ = versioning.create_registry_client(
             name=conftest.test_registry_client_name,
-            uri=conftest.registry_test_endpoints[0][0],
+            uri=conftest.registry_test_endpoints[0][0][0],
             description='who cares?'
         )