Merge f830264 into 394ce55

Makefile

@@ -185,6 +185,7 @@ dist_files = $(bdist_file) $(sdist_file)
 package_py_files := \
     $(wildcard $(package_name)/*.py) \
     $(wildcard $(package_name)/*/*.py) \
+		$(wildcard $(package_name)/*/*/*.py) \
 
 doc_help_source_files := \
     $(wildcard $(pywbemcli_module_path)/_cmd_*.py)
@@ -227,7 +228,7 @@ pylint_rc_file := pylintrc
 
 ifeq ($(PACKAGE_LEVEL),minimum)
   # Using pywbem 0.x
-  pylint_ignore := all_types_method_mock_v1.py,simple_mock_invokemethod_v1.py,py_err_processatstartup.py
+  pylint_ignore := all_types_method_mock_v1old.py,all_types_method_mock_v1new.py,simple_mock_invokemethod_v1old.py,simple_mock_invokemethod_v1new.py,py_err_processatstartup.py
 else
   # Using pywbem 1.x
   pylint_ignore := all_types_method_mock_v0.py,simple_mock_invokemethod_v0.py,py_err_processatstartup.py
@@ -284,6 +285,7 @@ dist_manifest_in_files := \
     *.py \
     $(package_name)/*.py \
     $(package_name)/*/*.py \
+		$(package_name)/*/*/*.py \
 
 # Files that are dependents of the distribution archive.
 # Keep in sync with dist_manifest_in_files.
@@ -295,6 +297,7 @@ dist_dependent_files := \
     $(wildcard *.py) \
     $(wildcard $(package_name)/*.py) \
     $(wildcard $(package_name)/*/*.py) \
+		$(wildcard $(package_name)/*/*/*.py) \
 
 # Scripts are required to install the OS-level components of pywbem.
 ifeq ($(PLATFORM),Windows_native)

docs/changes.rst

@@ -56,6 +56,23 @@ Released: not yet
 
 **Enhancements:**
 
+* Introduced caching of the mock environment used by connection definitions in
+  order to speed up the loading of the connection definition. The mock
+  environments are stored in directory ~/.pywbemcli_mockcache and are
+  automatically managed. The pywbemcli --verbose general option can be used
+  to show messages about the cache management. (See issue #689)
+
+* A new approach for the setup of mock scripts has been introduced: The mock
+  script defines a `setup(conn, server, verbose)` function that is called when
+  the mock environment is built. It is not called when the mock environment
+  is reinstantiated from the cache.
+  The old approach with setting global variables CONN, SERVER, VERBOSE is still
+  supported, but the mock environment cannot be cached and will be built every
+  time when mock scripts with that setup approach are used.
+  On Python <3.5, mock scripts with the `setup()` function are rejected, because
+  the functionality to import them is not available, and the compile+exec
+  approach does not allow executing the setup() function. (See issue #689)
+
 * Modify general help to display the full path of the default connections file.
   (See issue #660)
 

pylintrc

@@ -347,7 +347,7 @@ logging-modules=logging
 [SIMILARITIES]
 
 # Minimum lines number of a similarity.
-min-similarity-lines=20
+min-similarity-lines=40
 
 # Ignore comments when computing similarities.
 ignore-comments=yes

pywbemtools/pywbemcli/_connection_repository.py

@@ -31,29 +31,12 @@
 import click
 
 from ._pywbem_server import PywbemServer
+from ._pywbemcli_operations import delete_mock_cache
+from ._utils import DEFAULT_CONNECTIONS_FILE, B08_DEFAULT_CONNECTIONS_FILE
 
 if six.PY2:
     import codecs  # pylint: disable=wrong-import-order
 
-# Keep the following connections file definitions in sync with help text of
-# the "--connections-file" option in pywbemcli.py and generaloptions.rst, and
-# with the use of the base file name in several other .rst and .py files.
-
-# Base file name of the connections file
-# The B08_* file name was used before pywbemcli 0.8.
-CONNECTIONS_FILENAME = '.pywbemcli_connections.yaml'
-B08_CONNECTIONS_FILENAME = 'pywbemcli_connection_definitions.yaml'
-
-# Path name of default connections file directory.
-DEFAULT_CONNECTIONS_DIR = os.path.expanduser("~")
-
-# Path name of default connections file
-# The B08_* path name was used before pywbemcli 0.8.
-DEFAULT_CONNECTIONS_FILE = os.path.join(DEFAULT_CONNECTIONS_DIR,
-                                        CONNECTIONS_FILENAME)
-B08_DEFAULT_CONNECTIONS_FILE = os.path.join(DEFAULT_CONNECTIONS_DIR,
-                                            B08_CONNECTIONS_FILENAME)
-
 BAK_FILE_SUFFIX = 'bak'
 
 
@@ -159,7 +142,7 @@ class ConnectionRepository(object):
                 ca-certs: null
                 mock-server:
                 - tests/unit/simple_mock_model.mof
-                - tests/unit/simple_mock_invokemethod_v1.py
+                - tests/unit/simple_mock_invokemethod_v1old.py
             server1:
                 name: server1
                 server: https://woot.com
@@ -494,9 +477,10 @@ def _load_connections_file(self):
                 # Build the PywbemServer object for each connection definition
                 for name, svr in six.iteritems(connections_dict):
                     try:
-                        self._pywbemcli_servers[name] = \
-                            PywbemServer.create(
-                                replace_underscores=True, **svr)
+                        server = PywbemServer.create(
+                            replace_underscores=True,
+                            connections_file=self._connections_file,
+                            **svr)
                     except TypeError as te:
                         raise ConnectionsFileLoadError(
                             self._connections_file,
@@ -507,7 +491,7 @@ def _load_connections_file(self):
                             self._connections_file,
                             'Invalid attribute value in connection '
                             'definition "{0}": {1}'.format(name, ve))
-
+                    self._pywbemcli_servers[name] = server
                 self._loaded = True
 
         except (OSError, IOError) as exc:
@@ -562,6 +546,10 @@ def delete(self, name):
         """
         self._load_connections_file()
 
+        # Delete mock cache if it exists
+        delete_mock_cache(self._connections_file, name)
+
+        # Delete connection definition
         del self._pywbemcli_servers[name]
 
         # Unset default connection if it is the one being deleted

pywbemtools/pywbemcli/_pywbem_server.py

@@ -28,6 +28,7 @@
 import pywbem
 from pywbem import WBEMServer, configure_loggers_from_string
 
+from . import mockscripts
 from .config import DEFAULT_URL_SCHEME, DEFAULT_CONNECTION_TIMEOUT, \
     DEFAULT_NAMESPACE, MAX_TIMEOUT
 from ._pywbemcli_operations import PYWBEMCLIConnection, PYWBEMCLIFakedConnection
@@ -124,7 +125,7 @@ def __init__(self, server=None, default_namespace=DEFAULT_NAMESPACE,
                  name='default', user=None, password=None,
                  timeout=DEFAULT_CONNECTION_TIMEOUT, verify=None, use_pull=None,
                  pull_max_cnt=None, certfile=None, keyfile=None,
-                 ca_certs=None, mock_server=None):
+                 ca_certs=None, mock_server=None, connections_file=None):
         """
         Create  a PywbemServer object. This contains the configuration
         and operation information to create a connection to the server
@@ -153,9 +154,11 @@ def __init__(self, server=None, default_namespace=DEFAULT_NAMESPACE,
         self.keyfile = keyfile
         self.ca_certs = ca_certs
 
+        # May be None in case of not-saved connection (e.g. connection save)
+        self._connections_file = connections_file
+
         # dynamically created in create_connection
         self._wbem_server = None
-        self._conn = None
 
     def __str__(self):
         return 'PywbemServer(url={s.server} name={s.name})'.format(s=self)
@@ -403,16 +406,17 @@ def ca_certs(self, ca_certs):
     @property
     def conn(self):
         """
-        :class:`~pywbem.WBEMConnection` WBEMConnection to be used for requests.
+        :class:`PYWBEMCLIFakedConnection` or :class:`PYWBEMCLIConnection`
+        (both derived from :class:`pywbem.WBEMConnection`):
+        Connection object to be used for WBEM operation requests.
         """
-        # This is created in wbemserver and retained there.
         return self.wbem_server.conn if self.wbem_server else None
 
     @property
     def wbem_server(self):
         """
-        :class:`~pywbem.WBEMConnection` WBEMServer instance to be used for
-        requests.
+        :class:`~pywbem.WBEMServer`: Server object to be used for higher level
+        WBEM server requests.
         """
         return self._wbem_server
 
@@ -492,7 +496,6 @@ def reset(self):
         """ Reset the connection attributes of this pywbem server so that the
         connection must be restablished
         """
-        self._conn = None
         self._wbem_server = None
 
     def create_connection(self, log=None, use_pull=None,
@@ -512,27 +515,31 @@ def create_connection(self, log=None, use_pull=None,
             other pywbem cim_operation requests
 
         Raises:
-           ValueError: if server paramer is invalid or other issues with
+           ValueError: if server parameter is invalid or other issues with
            the input values
         """
+
         if self._mock_server:
-            if self.conn is None:
+            if self._wbem_server is None:
                 conn = PYWBEMCLIFakedConnection(
                     default_namespace=self.default_namespace,
                     use_pull_operations=use_pull,
                     stats_enabled=timestats)
+                self._wbem_server = WBEMServer(conn)
                 try:
-                    self._wbem_server = WBEMServer(conn)
-                    conn.build_repository(conn,
-                                          self._wbem_server,
-                                          self._mock_server,
-                                          verbose)
-                except click.Abort:
-                    raise
-                except Exception as exc:  # pylint: disable=broad-except
-                    click.echo(
-                        "Building the mock repository failed: {}".format(exc),
-                        err=True)
+                    conn.build_mockenv(
+                        self._wbem_server, self._mock_server,
+                        self._connections_file, self._name, verbose)
+                except (mockscripts.MockMOFCompileError,
+                        mockscripts.MockScriptError) as exc:
+                    # These errors cause the command to be aborted, because
+                    # partially executed mock scripts or partially compiled MOF
+                    # files might have caused inconsistencies in the Python
+                    # mockscripts namespace and in the CIM repository.
+                    click.echo(str(exc), err=True)
+                    raise click.Abort()
+                except mockscripts.MockError as exc:
+                    raise click.ClickException(str(exc))
 
         else:  # mock_server does not exist
             if not self.server: