Fixed Salt's deferred imports to allow onedir builds while not breaki…

…ng non-onedir builds

Fixes saltstack#146
Closes saltstack#144

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.pylint-spelling-words

@@ -107,6 +107,7 @@ nocover
 nodeid
 noqa
 nox
+onedir
 openbsd
 ordereddict
 os

changelog/146.bugfix.rst

@@ -0,0 +1,5 @@
+Fixed Salt's deferred imports to allow onedir builds while not breaking non-onedir builds:
+
+* Additionally, stopped relying on `salt.utils.files` and `salt.utils.yaml`
+* Stopped using `zmq` to forward events(this was where the breakage was showing) for a plain TCP implementation.
+* The `event_listener` fixture is now started/stopped like a regular pytest fixture

requirements/base.txt

@@ -8,5 +8,5 @@ pytest-shell-utilities>=1.4.0
 psutil
 pyyaml
 pyzmq
-msgpack
+msgpack>=0.5.2
 virtualenv

src/saltfactories/bases.py

@@ -593,7 +593,9 @@ def verify_config(cls, config):
         """
         Verify the configuration dictionary.
         """
-        import salt.utils.verify
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.utils.verify  # pylint: disable=import-outside-toplevel
 
         salt.utils.verify.verify_env(
             cls._get_verify_config_entries(config),

src/saltfactories/cli/cloud.py

@@ -26,6 +26,8 @@ def default_config(root_dir, master_id, defaults=None, overrides=None):
         """
         Return the default configuration for the daemon.
         """
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
         import salt.utils.dictupdate
 
         if defaults is None:
@@ -78,6 +80,8 @@ def verify_config(cls, config):
         """
         Verify the configuration dictionary.
         """
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
         import salt.config
         import salt.utils.verify
 

src/saltfactories/cli/key.py

@@ -30,9 +30,12 @@ def _default___cli_log_level_supported__(self):
         try:
             # Salt >= 3005
             return "--log-level" in SaltKeyOptionParser._console_log_level_cli_flags
-        except AttributeError:
+        except AttributeError:  # pragma: no cover
             # Salt <= 3004
-            return SaltKeyOptionParser._skip_console_logging_config_ is False
+            return (
+                SaltKeyOptionParser._skip_console_logging_config_  # pylint: disable=no-member
+                is False
+            )
 
     def get_minion_tgt(self, minion_tgt=None):
         """

src/saltfactories/cli/spm.py

@@ -34,7 +34,9 @@ def default_config(root_dir, master_factory, defaults=None, overrides=None):
         """
         Return the default configuration for the daemon.
         """
-        import salt.utils.dictupdate
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.utils.dictupdate  # pylint: disable=import-outside-toplevel
 
         if defaults is None:
             defaults = {}
@@ -89,8 +91,10 @@ def verify_config(cls, config):
         """
         Verify the configuration dictionary.
         """
-        import salt.config
-        import salt.utils.verify
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
+        import salt.utils.verify  # pylint: disable=import-outside-toplevel
 
         prepend_root_dirs = [
             "formula_path",
@@ -127,8 +131,10 @@ def write_config(cls, config):
         """
         Verify the loaded configuration.
         """
-        import salt.config
-        import salt.utils.verify
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
+        import salt.utils.verify  # pylint: disable=import-outside-toplevel
 
         cls.verify_config(config)
         config_file = config.pop("spm_conf_file")

src/saltfactories/client.py

@@ -39,7 +39,9 @@ def _set_functions_known_to_return_none(self):
 
     @__client.default
     def _set_client(self):
-        import salt.client
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.client  # pylint: disable=import-outside-toplevel
 
         return salt.client.get_local_client(mopts=self.master_config)
 

src/saltfactories/daemons/master.py

@@ -69,8 +69,10 @@ def default_config(
         """
         Return the default configuration.
         """
-        import salt.config
-        import salt.utils.dictupdate
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
+        import salt.utils.dictupdate  # pylint: disable=import-outside-toplevel
 
         if defaults is None:
             defaults = {}
@@ -274,7 +276,9 @@ def load_config(cls, config_file, config):
         """
         Return the loaded configuration.
         """
-        import salt.config
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
 
         return salt.config.master_config(config_file)
 

src/saltfactories/daemons/minion.py

@@ -49,8 +49,10 @@ def default_config(
         """
         Return the default configuration.
         """
-        import salt.config
-        import salt.utils.dictupdate
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
+        import salt.utils.dictupdate  # pylint: disable=import-outside-toplevel
 
         if defaults is None:
             defaults = {}
@@ -224,7 +226,9 @@ def load_config(cls, config_file, config):
         """
         Return the loaded configuration.
         """
-        import salt.config
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
 
         return salt.config.minion_config(config_file, minion_id=config["id"], cache_minion_id=True)
 

src/saltfactories/daemons/proxy.py

@@ -71,8 +71,10 @@ def default_config(
         """
         Return the default configuration.
         """
-        import salt.config
-        import salt.utils.dictupdate
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
+        import salt.utils.dictupdate  # pylint: disable=import-outside-toplevel
 
         if defaults is None:
             defaults = {}
@@ -240,7 +242,9 @@ def load_config(cls, config_file, config):
         """
         Return the loaded configuration.
         """
-        import salt.config
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
 
         return salt.config.proxy_config(config_file, minion_id=config["id"], cache_minion_id=True)
 

src/saltfactories/daemons/syndic.py

@@ -34,8 +34,10 @@ def default_config(
         """
         Return the default configuration.
         """
-        import salt.config
-        import salt.utils.dictupdate
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
+        import salt.utils.dictupdate  # pylint: disable=import-outside-toplevel
 
         if defaults is None:
             defaults = {}
@@ -143,7 +145,9 @@ def load_config(cls, config_file, config):
         """
         Return the loaded configuration.
         """
-        import salt.config
+        # Do not move these deferred imports. It allows running against a Salt
+        # onedir build in salt's repo checkout.
+        import salt.config  # pylint: disable=import-outside-toplevel
 
         conf_dir = pathlib.Path(config_file).parent.parent
         master_config_file = str(conf_dir / "master")

src/saltfactories/manager.py

@@ -162,7 +162,10 @@ def final_minion_config_tweaks(self, config):
         pytest_key = "pytest-minion"
         if pytest_key not in config:  # pragma: no cover
             config[pytest_key] = {}
-        config[pytest_key]["returner_address"] = self.event_listener.address
+        config[pytest_key]["returner_address"] = {
+            "host": self.event_listener.host,
+            "port": self.event_listener.port,
+        }
         self.final_common_config_tweaks(config, "minion")
 
     def final_master_config_tweaks(self, config):
@@ -172,7 +175,10 @@ def final_master_config_tweaks(self, config):
         pytest_key = "pytest-master"
         if pytest_key not in config:  # pragma: no cover
             config[pytest_key] = {}
-        config[pytest_key]["returner_address"] = self.event_listener.address
+        config[pytest_key]["returner_address"] = {
+            "host": self.event_listener.host,
+            "port": self.event_listener.port,
+        }
         self.final_common_config_tweaks(config, "master")
 
     def final_syndic_config_tweaks(self, config):

src/saltfactories/plugins/__init__.py

@@ -80,7 +80,9 @@ def pytest_load_initial_conftests(*_):
     """
     Register our pytest helpers.
     """
-    import salt.version
+    # Do not move these deferred imports. It allows running against a Salt
+    # onedir build in salt's repo checkout.
+    import salt.version  # pylint: disable=import-outside-toplevel
 
     if salt.version.__saltstack_version__ < "3004":
         raise pytest.UsageError("Only salt>=3004 is supported")