MongoDB: Add wrapper, base class and tests

- Public API of AbstractDB class consists of: `initiate_connection`,
`is_connected`, `close_connection`, `write`, `read` and `remove`.
- `read` uses `MongoClient.find`
- `write` uses `MongoClient.insert_many`, when `query` is not supplied,
and `MongoClient.update_many` with `upsert=True` else.
- `remove` uses `MongoClient.delete_many`.
- Initializing `MongoDB` class needs either a proper Mongo URI or
separate args, which overwrite elements infered from a possible URI.
- Create tests for each interface cases.
- AbstractDB is an singleton.
- Fix docs configuration.

.pylintrc

@@ -51,7 +51,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=fixme,F0401,intern-builtin,nonzero-method,parameter-unpacking,backtick,raw_input-builtin,dict-view-method,filter-builtin-not-iterating,long-builtin,unichr-builtin,input-builtin,unicode-builtin,file-builtin,map-builtin-not-iterating,delslice-method,apply-builtin,cmp-method,setslice-method,coerce-method,long-suffix,raising-string,import-star-module-level,buffer-builtin,reload-builtin,unpacking-in-except,print-statement,hex-method,old-octal-literal,metaclass-assignment,dict-iter-method,range-builtin-not-iterating,using-cmp-argument,indexing-exception,no-absolute-import,coerce-builtin,getslice-method,suppressed-message,execfile-builtin,round-builtin,useless-suppression,reduce-builtin,old-raise-syntax,zip-builtin-not-iterating,cmp-builtin,xrange-builtin,standarderror-builtin,old-division,oct-method,next-method-called,old-ne-operator,basestring-builtin
+disable=unidiomatic-typecheck,redefined-outer-name,fixme,F0401,intern-builtin,wrong-import-position
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -375,7 +375,7 @@ known-third-party=enchant
 max-args=8
 
 # Maximum number of attributes for a class (see R0902).
-max-attributes=7
+max-attributes=10
 
 # Maximum number of boolean expressions in a if statement
 max-bool-expr=5

.travis.yml

@@ -29,7 +29,7 @@ install:
   - pip install tox
 before_script:
   - sleep 15
-  - mongo metaopt --eval 'db.createUser({user:"user",pwd:"pass",roles:["readWrite"]});'
+  - mongo metaopt_test --eval 'db.createUser({user:"user",pwd:"pass",roles:["readWrite"]});'
 script:
   - tox
 after_success:

MANIFEST.in

@@ -21,4 +21,4 @@ include docs/requirements.txt
 prune docs/build
 prune docs/src/reference
 recursive-include src *.py
-recursive-include tests *.py
+recursive-include tests *.py *.yaml

codecov.yml

@@ -21,9 +21,9 @@ coverage:
     project:
       # add more later when packages will be namespaced
       default: false
-      tests:
-        target: 100%
-        paths: tests
+      # tests:
+      #   target: 100%
+      #   paths: tests
       core:
         # change this to more strict number and some allowance threshold later
         target: auto

docs/requirements.txt

@@ -1 +1,2 @@
 sphinx >= 1.6.5, < 2
+numpydoc

docs/src/conf.py

@@ -15,9 +15,19 @@
 If extensions (or modules to document with autodoc) are in another directory,
 add these directories to sys.path here. If the directory is relative to the
 documentation root, use os.path.abspath to make it absolute, like shown here.
+
+How to document -- sources:
+
+   1. `Numpy Standard <https://numpydoc.readthedocs.io/en/latest/format.html>`_
+   2. `Python Standard <https://docs.python.org/devguide/documenting.html>`_
+   3. `reST general <http://www.sphinx-doc.org/en/stable/rest.html>`_
+   4. `reST reference tags <http://www.sphinx-doc.org/en/stable/domains.html#the-python-domain>`_
+   5. `Cross-reference <http://www.sphinx-doc.org/en/stable/domains.html#python-roles>`_
+
 """
 from __future__ import absolute_import
 
+import glob
 import os
 import re
 import sys
@@ -44,7 +54,9 @@
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
-    'sphinx.ext.extlinks'
+    'sphinx.ext.extlinks',
+    'sphinx.ext.autosummary',
+    'numpydoc',
 ]
 
 # General information about the project.
@@ -92,6 +104,9 @@
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
 
+# The reST default role (used for this markup: `text`) to use for all documents.
+default_role = "autolink"
+
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -181,4 +196,19 @@
 
 # -- Autodoc configuration -----------------------------------------------
 
-autodoc_mock_imports = ['_version', 'utils.appdirs']
+autodoc_mock_imports = ['_version', 'utils._appdirs']
+
+################################################################################
+#                             Numpy Doc Extension                              #
+################################################################################
+
+# sphinx.ext.autosummary will automatically be loaded as well. So:
+autosummary_generate = glob.glob("reference/*.rst")
+
+# Generate ``plot::`` for ``Examples`` sections which contain matplotlib
+numpydoc_use_plots = False
+
+# Create a Sphinx table of contents for the lists of class methods and
+# attributes. If a table of contents is made, Sphinx expects each entry to have
+# a separate page.
+numpydoc_class_members_toctree = False

setup.py

@@ -54,7 +54,7 @@ def find_data_files():
             'mopt = metaopt.cli:main',
             ],
         },
-    install_requires=['six', 'PyYAML'],
+    install_requires=['six', 'PyYAML', 'pymongo>=3'],
     tests_require=['pytest>=3.0.0'],
     setup_requires=['setuptools', 'pytest-runner>=2.0,<3dev'],
     #  http://peak.telecommunity.com/DevCenter/setuptools#setting-the-zip-safe-flag

src/metaopt/io/database/__init__.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+"""
+:mod:`metaopt.io.database` -- Wrappers for database frameworks
+==============================================================
+
+.. module:: database
+   :platform: Unix
+   :synopsis: Import name for wrappers of database frameworks.
+
+Contains :class:`AbstractDB`, an interface for databases.
+Currently, implemented wrappers:
+
+   - :class:`metaopt.io.database.mongodb.MongoDB`
+
+"""
+from __future__ import absolute_import
+
+from abc import abstractmethod, abstractproperty
+import logging
+
+import six
+
+from metaopt.utils import AbstractSingletonType
+
+
+@six.add_metaclass(AbstractSingletonType)
+class AbstractDB(object):
+    """Base class for database framework wrappers.
+
+    Attributes
+    ----------
+    host : str
+       It can be either:
+          1. Known hostname or IP address in which database server resides.
+          2. URI: A database framework specific connection string.
+    dbname : str
+       Name of database containing experiments.
+    port : int
+       Port that database server listens to for requests.
+    username : str
+        Name of user with write/read permissions to database with name `dbname`.
+    password : str
+        Secret phrase of user, `username`.
+
+    """
+
+    def __init__(self, host='localhost', dbname=None,
+                 port=None, username=None, password=None):
+        """Init method, see attributes of :class:`AbstractDB`."""
+        self._logger = logging.getLogger(__name__)
+
+        self.host = host
+        self.dbname = dbname
+        self.port = port
+        self.username = username
+        self.password = password
+
+        self._db = None
+        self._conn = None
+        self.initiate_connection()
+
+    @abstractproperty
+    def is_connected(self):
+        """True, if practical connection has been achieved."""
+        pass
+
+    @abstractmethod
+    def initiate_connection(self):
+        """Connect to database, unless `AbstractDB` `is_connected`.
+
+        :raises :exc:`DatabaseError`: if connection or authentication fails
+
+        """
+        pass
+
+    @abstractmethod
+    def close_connection(self):
+        """Disconnect from database, if `AbstractDB` `is_connected`."""
+        pass
+
+    @abstractmethod
+    def write(self, collection_name, data,
+              query=None):
+        """Write new information to a collection. Perform insert or update.
+
+        Parameters
+        ----------
+        collection_name : str
+           A collection inside database, a table.
+        data : dict or list of dicts
+           New data that will **be inserted** or that will **update** entries.
+        query : dict, optional
+           Assumes an update operation: filter entries in collection to be updated.
+
+        :return: operation success.
+
+        .. note::
+           In the case of an insert operation, `data` variable will be updated
+           to contain a unique *_id* key.
+
+        .. note::
+           In the case of an update operation, if `query` fails to find a
+           document that matches, insert of `data` will be performed instead.
+
+        """
+        pass
+
+    @abstractmethod
+    def read(self, collection_name, query, selection=None):
+        """Read a collection and return a value according to the query.
+
+        Parameters
+        ----------
+        collection_name : str
+           A collection inside database, a table.
+        query : dict
+           Filter entries in collection.
+        selection : dict, optional
+           Elements of matched entries to return, the projection.
+
+        :return: matched document[s]
+
+        """
+        pass
+
+    @abstractmethod
+    def remove(self, collection_name, query):
+        """Delete from a collection document[s] which match the `query`.
+
+        Parameters
+        ----------
+        collection_name : str
+           A collection inside database, a table.
+        query : dict
+           Filter entries in collection.
+
+        :return: operation success.
+
+        """
+        pass
+
+
+class DatabaseError(RuntimeError):
+    """Exception type used to delegate responsibility from any database
+    implementation's own Exception types.
+    """
+
+    pass
+
+
+from metaopt.io.database.mongodb import MongoDB  # noqa