PluginStorage to store variables between invocations. (#5468)

The base class for Installer plugins `certbot.plugins.common.Installer` now provides functionality of `PluginStorage` to all installer plugins. This allows a plugin to save and retrieve variables in between of invocations.

The on disk storage is basically a JSON file at `config_dir`/`.pluginstorage.json`, usually `/etc/letsencrypt/.pluginstorage.json`. The JSON structure is automatically namespaced using the internal plugin name as a namespace key. Because the actual storage is JSON, the supported data types are: dict, list, tuple, str, unicode, int, long, float, boolean and nonetype.

To add a variable from inside the plugin class:
`self.storage.put("my_variable_name", my_var)`

To fetch a variable from inside the plugin class:
`my_var = self.storage.fetch("my_variable_key")`

The storage state isn't written on disk automatically, but needs to be called:
`self.storage.save()`

* Plugin storage implementation

* Added config_dir to existing test mocks

* PluginStorage test cases

* Saner handling of bad config_dir paths

* Storage moved to Installer and not initialized on plugin __init__

* Finetuning and renaming

certbot/errors.py

@@ -87,6 +87,10 @@ class NotSupportedError(PluginError):
     """Certbot Plugin function not supported error."""
 
 
+class PluginStorageError(PluginError):
+    """Certbot Plugin Storage error."""
+
+
 class StandaloneBindError(Error):
     """Standalone plugin bind error."""
 

certbot/plugins/common.py

@@ -18,6 +18,8 @@
 from certbot import reverter
 from certbot import util
 
+from certbot.plugins.storage import PluginStorage
+
 logger = logging.getLogger(__name__)
 
 
@@ -99,7 +101,6 @@ def dest(self, var):
     def conf(self, var):
         """Find a configuration value for variable ``var``."""
         return getattr(self.config, self.dest(var))
-# other
 
 
 class Installer(Plugin):
@@ -110,6 +111,7 @@ class Installer(Plugin):
     """
     def __init__(self, *args, **kwargs):
         super(Installer, self).__init__(*args, **kwargs)
+        self.storage = PluginStorage(self.config, self.name)
         self.reverter = reverter.Reverter(self.config)
 
     def add_to_checkpoint(self, save_files, save_notes, temporary=False):

certbot/plugins/storage.py

@@ -0,0 +1,117 @@
+"""Plugin storage class."""
+import json
+import logging
+import os
+
+from certbot import errors
+
+logger = logging.getLogger(__name__)
+
+class PluginStorage(object):
+    """Class implementing storage functionality for plugins"""
+
+    def __init__(self, config, classkey):
+        """Initializes PluginStorage object storing required configuration
+        options.
+
+        :param .configuration.NamespaceConfig config: Configuration object
+        :param str classkey: class name to use as root key in storage file
+
+        """
+
+        self._config = config
+        self._classkey = classkey
+        self._initialized = False
+        self._data = None
+        self._storagepath = None
+
+    def _initialize_storage(self):
+        """Initializes PluginStorage data and reads current state from the disk
+        if the storage json exists."""
+
+        self._storagepath = os.path.join(self._config.config_dir, ".pluginstorage.json")
+        self._load()
+        self._initialized = True
+
+    def _load(self):
+        """Reads PluginStorage content from the disk to a dict structure
+
+        :raises .errors.PluginStorageError: when unable to open or read the file
+        """
+        data = dict()
+        filedata = ""
+        try:
+            with open(self._storagepath, 'r') as fh:
+                filedata = fh.read()
+        except IOError as e:
+            errmsg = "Could not read PluginStorage data file: {0} : {1}".format(
+                self._storagepath, str(e))
+            if os.path.isfile(self._storagepath):
+                # Only error out if file exists, but cannot be read
+                logger.error(errmsg)
+                raise errors.PluginStorageError(errmsg)
+        try:
+            data = json.loads(filedata)
+        except ValueError:
+            if not filedata:
+                logger.debug("Plugin storage file %s was empty, no values loaded",
+                             self._storagepath)
+            else:
+                errmsg = "PluginStorage file {0} is corrupted.".format(
+                    self._storagepath)
+                logger.error(errmsg)
+                raise errors.PluginStorageError(errmsg)
+        self._data = data
+
+    def save(self):
+        """Saves PluginStorage content to disk
+
+        :raises .errors.PluginStorageError: when unable to serialize the data
+            or write it to the filesystem
+        """
+        if not self._initialized:
+            errmsg = "Unable to save, no values have been added to PluginStorage."
+            logger.error(errmsg)
+            raise errors.PluginStorageError(errmsg)
+
+        try:
+            serialized = json.dumps(self._data)
+        except TypeError as e:
+            errmsg = "Could not serialize PluginStorage data: {0}".format(
+                str(e))
+            logger.error(errmsg)
+            raise errors.PluginStorageError(errmsg)
+        try:
+            with os.fdopen(os.open(self._storagepath,
+                                   os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fh:
+                fh.write(serialized)
+        except IOError as e:
+            errmsg = "Could not write PluginStorage data to file {0} : {1}".format(
+                self._storagepath, str(e))
+            logger.error(errmsg)
+            raise errors.PluginStorageError(errmsg)
+
+    def put(self, key, value):
+        """Put configuration value to PluginStorage
+
+        :param str key: Key to store the value to
+        :param value: Data to store
+        """
+        if not self._initialized:
+            self._initialize_storage()
+
+        if not self._classkey in self._data.keys():
+            self._data[self._classkey] = dict()
+        self._data[self._classkey][key] = value
+
+    def fetch(self, key):
+        """Get configuration value from PluginStorage
+
+        :param str key: Key to get value from the storage
+
+        :raises KeyError: If the key doesn't exist in the storage
+        """
+        if not self._initialized:
+            self._initialize_storage()
+
+        return self._data[self._classkey][key]

certbot/plugins/storage_test.py

@@ -0,0 +1,117 @@
+"""Tests for certbot.plugins.storage.PluginStorage"""
+import json
+import mock
+import os
+import unittest
+
+from certbot import errors
+
+from certbot.plugins import common
+from certbot.tests import util as test_util
+
+class PluginStorageTest(test_util.ConfigTestCase):
+    """Test for certbot.plugins.storage.PluginStorage"""
+
+    def setUp(self):
+        super(PluginStorageTest, self).setUp()
+        self.plugin_cls = common.Installer
+        os.mkdir(self.config.config_dir)
+        with mock.patch("certbot.reverter.util"):
+            self.plugin = self.plugin_cls(config=self.config, name="mockplugin")
+
+    def test_load_errors_cant_read(self):
+        with open(os.path.join(self.config.config_dir,
+                               ".pluginstorage.json"), "w") as fh:
+            fh.write("dummy")
+        # When unable to read file that exists
+        mock_open = mock.mock_open()
+        mock_open.side_effect = IOError
+        self.plugin.storage.storagepath = os.path.join(self.config.config_dir,
+                                                       ".pluginstorage.json")
+        with mock.patch("six.moves.builtins.open", mock_open):
+            with mock.patch('os.path.isfile', return_value=True):
+                with mock.patch("certbot.reverter.util"):
+                    self.assertRaises(errors.PluginStorageError,
+                                      self.plugin.storage._load)  # pylint: disable=protected-access
+
+    def test_load_errors_empty(self):
+        with open(os.path.join(self.config.config_dir, ".pluginstorage.json"), "w") as fh:
+            fh.write('')
+        with mock.patch("certbot.plugins.storage.logger.debug") as mock_log:
+            # Should not error out but write a debug log line instead
+            with mock.patch("certbot.reverter.util"):
+                nocontent = self.plugin_cls(self.config, "mockplugin")
+            self.assertRaises(KeyError,
+                              nocontent.storage.fetch, "value")
+            self.assertTrue(mock_log.called)
+            self.assertTrue("no values loaded" in mock_log.call_args[0][0])
+
+    def test_load_errors_corrupted(self):
+        with open(os.path.join(self.config.config_dir,
+                               ".pluginstorage.json"), "w") as fh:
+            fh.write('invalid json')
+        with mock.patch("certbot.plugins.storage.logger.error") as mock_log:
+            with mock.patch("certbot.reverter.util"):
+                corrupted = self.plugin_cls(self.config, "mockplugin")
+            self.assertRaises(errors.PluginError,
+                              corrupted.storage.fetch,
+                              "value")
+            self.assertTrue("is corrupted" in mock_log.call_args[0][0])
+
+    def test_save_errors_cant_serialize(self):
+        with mock.patch("certbot.plugins.storage.logger.error") as mock_log:
+            # Set data as something that can't be serialized
+            self.plugin.storage._initialized = True  # pylint: disable=protected-access
+            self.plugin.storage.storagepath = "/tmp/whatever"
+            self.plugin.storage._data = self.plugin_cls  # pylint: disable=protected-access
+            self.assertRaises(errors.PluginStorageError,
+                              self.plugin.storage.save)
+            self.assertTrue("Could not serialize" in mock_log.call_args[0][0])
+
+    def test_save_errors_unable_to_write_file(self):
+        mock_open = mock.mock_open()
+        mock_open.side_effect = IOError
+        with mock.patch("os.open", mock_open):
+            with mock.patch("certbot.plugins.storage.logger.error") as mock_log:
+                self.plugin.storage._data = {"valid": "data"}  # pylint: disable=protected-access
+                self.plugin.storage._initialized = True  # pylint: disable=protected-access
+                self.assertRaises(errors.PluginStorageError,
+                                  self.plugin.storage.save)
+                self.assertTrue("Could not write" in mock_log.call_args[0][0])
+
+    def test_save_uninitialized(self):
+        with mock.patch("certbot.reverter.util"):
+            self.assertRaises(errors.PluginStorageError,
+                              self.plugin_cls(self.config, "x").storage.save)
+
+    def test_namespace_isolation(self):
+        with mock.patch("certbot.reverter.util"):
+            plugin1 = self.plugin_cls(self.config, "first")
+            plugin2 = self.plugin_cls(self.config, "second")
+        plugin1.storage.put("first_key", "first_value")
+        self.assertRaises(KeyError,
+                          plugin2.storage.fetch, "first_key")
+        self.assertRaises(KeyError,
+                          plugin2.storage.fetch, "first")
+        self.assertEqual(plugin1.storage.fetch("first_key"), "first_value")
+
+
+    def test_saved_state(self):
+        self.plugin.storage.put("testkey", "testvalue")
+        # Write to disk
+        self.plugin.storage.save()
+        with mock.patch("certbot.reverter.util"):
+            another = self.plugin_cls(self.config, "mockplugin")
+        self.assertEqual(another.storage.fetch("testkey"), "testvalue")
+
+        with open(os.path.join(self.config.config_dir,
+                               ".pluginstorage.json"), 'r') as fh:
+            psdata = fh.read()
+        psjson = json.loads(psdata)
+        self.assertTrue("mockplugin" in psjson.keys())
+        self.assertEqual(len(psjson), 1)
+        self.assertEqual(psjson["mockplugin"]["testkey"], "testvalue")
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover