Add api of registering storage

dolphin/api/schemas/storages.py

@@ -0,0 +1,38 @@
+# Copyright 2020 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dolphin.api.validation import parameter_types
+
+
+create = {
+    'type': 'object',
+    'properties': {
+        'host': parameter_types.hostname_or_ip_address,
+        'port': parameter_types.tcp_udp_port,
+        'username': {'type': 'string', 'minLength': 1, 'maxLength': 255},
+        'password': {'type': 'string', 'minLength': 1, 'maxLength': 255},
+        'vendor': {'type': 'string', 'minLength': 1, 'maxLength': 255},
+        'model': {'type': 'string', 'minLength': 1, 'maxLength': 255},
+        'extra_attributes': {
+            'type': 'object',
+            'patternProperties': {
+                '^[a-zA-Z0-9-_:. ]{1,255}$': {
+                    'type': 'string', 'maxLength': 255
+                }
+            }
+        }
+    },
+    'required': ['host', 'port', 'username', 'password', 'vendor', 'model'],
+    'additionalProperties': False
+}

dolphin/api/v1/storages.py

@@ -12,23 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import six
 from six.moves import http_client
 import webob
 from webob import exc
 
 from oslo_log import log
 
-from dolphin import db, context
-from dolphin.api.views import storages as storage_view
 from dolphin.api.common import wsgi
+from dolphin.api.schemas import storages as schema_storages
+from dolphin.api import validation
+from dolphin.api.views import storages as storage_view
+from dolphin import context
+from dolphin import coordination
+from dolphin import cryptor
+from dolphin import db
 from dolphin.drivers import manager as drivermanager
-from dolphin.db.sqlalchemy import api as db
 from dolphin import exception
-from dolphin import utils
 from dolphin.i18n import _
-from dolphin import context
 from dolphin.task_manager import rpcapi as task_rpcapi
+from dolphin import utils
 
 LOG = log.getLogger(__name__)
 
@@ -51,7 +55,9 @@ def validate_parameters(data, required_parameters,
 
 class StorageController(wsgi.Controller):
     def __init__(self):
+        super().__init__()
         self.task_rpcapi = task_rpcapi.TaskAPI()
+        self.driver_manager = drivermanager.DriverManager()
 
     def index(self, req):
 
@@ -85,72 +91,37 @@ def index(self, req):
     def show(self, req, id):
         return dict(name="Storage 2")
 
+    @wsgi.response(201)
+    @validation.schema(schema_storages.create)
+    @coordination.synchronized('storage-create-{body[host]}-{body[port]}')
     def create(self, req, body):
-        """
-        This function for registering the new storage device
-        :param req:
-        :param body: "It contains the all input parameters"
-        :return:
-        """
-        # Check if body is valid
-        if not self.is_valid_body(body, 'storages'):
-            msg = _("Storage entity not found in request body")
-            raise exc.HTTPUnprocessableEntity(explanation=msg)
-
-        storage = body['storages']
-
-        # validate the body has all required parameters
-        required_parameters = ('hostip', 'vendor', 'model', 'username',
-                               'password')
-        validate_parameters(storage, required_parameters)
-
-        # validate the hostip
-        if not utils.is_valid_ip_address(storage['hostip'], ip_version='4'):
-            msg = _("Invalid hostip: {0}. Please provide a "
-                    "valid hostip".format(storage['hostip']))
-            LOG.error(msg)
-            raise exception.InvalidHost(msg)
+        """Register a new storage device."""
+        ctxt = req.environ['dolphin.context']
+        access_info_dict = body
 
-        # get dolphin.context. Later may be validated context parameters
-        context = req.environ.get('dolphin.context')
+        if self._is_registered(ctxt, access_info_dict):
+            msg = _("Storage has been registered.")
+            raise exc.HTTPBadRequest(explanation=msg)
 
-        driver = drivermanager.DriverManager()
         try:
-            device_info = driver.register_storage(context, storage)
-            status = ''
-            if device_info.get('status') == 'available':
-                status = device_info.get('status')
-        except AttributeError as e:
-            LOG.error(e)
-            raise exception.DolphinException(e)
+            storage = self.driver_manager.register_storage(ctxt, access_info_dict)
+            storage = db.storage_create(context, storage)
+
+            # Need to encode the password before saving.
+            access_info_dict['storage_id'] = storage['id']
+            access_info_dict['password'] = cryptor.encode(access_info_dict['password'])
+            db.access_info_create(context, access_info_dict)
+        except (exception.InvalidCredential,
+                exception.StorageDriverNotFound,
+                exception.AccessInfoNotFound,
+                exception.StorageNotFound) as e:
+            raise exc.HTTPBadRequest(explanation=e.message)
         except Exception as e:
-            msg = _('Failed to register device in driver :{0}'.format(e))
-            LOG.error(e)
-            raise exception.DolphinException(msg)
-
-        if status == 'available':
-            try:
-                storage['storage_id'] = device_info.get('id')
-
-                db.access_info_create(context, storage)
-
-                db.storage_create(context, device_info)
-            except AttributeError as e:
-                LOG.error(e)
-                raise exception.DolphinException(e)
-            except Exception as e:
-                msg = _('Failed to create device entry in DB: {0}'
-                        .format(e))
-                LOG.exception(msg)
-                raise exception.DolphinException(msg)
-
-        else:
-            msg = _('Device registration failed with status: {0}'
-                    .format(status))
+            msg = _('Failed to register storage: {0}'.format(e))
             LOG.error(msg)
-            raise exception.DolphinException(msg)
+            raise exc.HTTPBadRequest(explanation=msg)
 
-        return device_info
+        return storage_view.build_storage(storage)
 
     def update(self, req, id, body):
         return dict(name="Storage 4")
@@ -185,6 +156,21 @@ def sync(self, req, id):
 
         return dict(name="Sync storage 1")
 
+    def _is_registered(self, context, access_info):
+        access_info_dict = copy.deepcopy(access_info)
+
+        # Remove unrelated query fields
+        access_info_dict.pop('username', None)
+        access_info_dict.pop('password', None)
+        access_info_dict.pop('vendor', None)
+        access_info_dict.pop('model', None)
+
+        # Check if storage is registered
+        if db.access_info_get_all(context,
+                                  filters=access_info_dict):
+            return True
+        return False
+
 
 def create_resource():
     return wsgi.Resource(StorageController())

dolphin/api/validation/parameter_types.py

@@ -159,3 +159,16 @@ def valid_char(char):
     'type': ['string', 'null'], 'minLength': 0, 'maxLength': 255,
     'pattern': valid_description_regex,
 }
+
+hostname_or_ip_address = {
+    # NOTE: Allow to specify hostname, ipv4 and ipv6.
+    'type': 'string', 'minLength': 0, 'maxLength': 255,
+    'pattern': '^[a-zA-Z0-9-_.:]*$'
+}
+
+tcp_udp_port = {
+    'type': ['integer', 'string'],
+    'pattern': '^[0-9]*$',
+    'minimum': 0, 'maximum': 65535
+}
+

dolphin/common/sqlalchemyutils.py

@@ -0,0 +1,172 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2010-2011 OpenStack Foundation
+# Copyright 2012 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Implementation of paginate query."""
+import datetime
+
+from oslo_log import log as logging
+from six.moves import range
+import sqlalchemy
+import sqlalchemy.sql as sa_sql
+from sqlalchemy.sql import type_api
+
+from dolphin.db import api
+from dolphin import exception
+from dolphin.i18n import _
+
+
+LOG = logging.getLogger(__name__)
+
+_TYPE_SCHEMA = {
+    'datetime': datetime.datetime(1900, 1, 1),
+    'big_integer': 0,
+    'integer': 0,
+    'string': ''
+}
+
+
+def _get_default_column_value(model, column_name):
+    """Return the default value of the columns from DB table.
+
+    In postgreDB case, if no right default values are being set, an
+    psycopg2.DataError will be thrown.
+    """
+    attr = getattr(model, column_name)
+    # Return the default value directly if the model contains. Otherwise return
+    # a default value which is not None.
+    if attr.default and isinstance(attr.default, type_api.TypeEngine):
+        return attr.default.arg
+
+    attr_type = attr.type
+    return _TYPE_SCHEMA[attr_type.__visit_name__]
+
+
+# TODO(wangxiyuan): Use oslo_db.sqlalchemy.utils.paginate_query once it is
+# stable and afforded by the minimum version in requirement.txt.
+# copied from glance/db/sqlalchemy/api.py
+def paginate_query(query, model, limit, sort_keys, marker=None,
+                   sort_dir=None, sort_dirs=None, offset=None):
+    """Returns a query with sorting / pagination criteria added.
+
+    Pagination works by requiring a unique sort_key, specified by sort_keys.
+    (If sort_keys is not unique, then we risk looping through values.)
+    We use the last row in the previous page as the 'marker' for pagination.
+    So we must return values that follow the passed marker in the order.
+    With a single-valued sort_key, this would be easy: sort_key > X.
+    With a compound-values sort_key, (k1, k2, k3) we must do this to repeat
+    the lexicographical ordering:
+    (k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3)
+
+    We also have to cope with different sort_directions.
+
+    Typically, the id of the last row is used as the client-facing pagination
+    marker, then the actual marker object must be fetched from the db and
+    passed in to us as marker.
+
+    :param query: the query object to which we should add paging/sorting
+    :param model: the ORM model class
+    :param limit: maximum number of items to return
+    :param sort_keys: array of attributes by which results should be sorted
+    :param marker: the last item of the previous page; we returns the next
+                    results after this value.
+    :param sort_dir: direction in which results should be sorted (asc, desc)
+    :param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys
+    :param offset: the number of items to skip from the marker or from the
+                    first element.
+
+    :rtype: sqlalchemy.orm.query.Query
+    :return: The query with sorting/pagination added.
+    """
+
+    if sort_dir and sort_dirs:
+        raise AssertionError('Both sort_dir and sort_dirs specified.')
+
+    # Default the sort direction to ascending
+    if sort_dirs is None and sort_dir is None:
+        sort_dir = 'asc'
+
+    # Ensure a per-column sort direction
+    if sort_dirs is None:
+        sort_dirs = [sort_dir for _sort_key in sort_keys]
+
+    if len(sort_dirs) != len(sort_keys):
+        raise AssertionError(
+            'sort_dirs length is not equal to sort_keys length.')
+
+    # Add sorting
+    for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs):
+        sort_dir_func = {
+            'asc': sqlalchemy.asc,
+            'desc': sqlalchemy.desc,
+        }[current_sort_dir]
+
+        try:
+            sort_key_attr = getattr(model, current_sort_key)
+        except AttributeError:
+            raise exception.InvalidInput(reason='Invalid sort key')
+        if not api.is_orm_value(sort_key_attr):
+            raise exception.InvalidInput(reason='Invalid sort key')
+        query = query.order_by(sort_dir_func(sort_key_attr))
+
+    # Add pagination
+    if marker is not None:
+        marker_values = []
+        for sort_key in sort_keys:
+            v = getattr(marker, sort_key)
+            if v is None:
+                v = _get_default_column_value(model, sort_key)
+            marker_values.append(v)
+
+        # Build up an array of sort criteria as in the docstring
+        criteria_list = []
+        for i in range(0, len(sort_keys)):
+            crit_attrs = []
+            for j in range(0, i):
+                model_attr = getattr(model, sort_keys[j])
+                default = _get_default_column_value(model, sort_keys[j])
+                attr = sa_sql.expression.case([(model_attr.isnot(None),
+                                                model_attr), ],
+                                              else_=default)
+                crit_attrs.append((attr == marker_values[j]))
+
+            model_attr = getattr(model, sort_keys[i])
+            default = _get_default_column_value(model, sort_keys[i])
+            attr = sa_sql.expression.case([(model_attr.isnot(None),
+                                            model_attr), ],
+                                          else_=default)
+            if sort_dirs[i] == 'desc':
+                crit_attrs.append((attr < marker_values[i]))
+            elif sort_dirs[i] == 'asc':
+                crit_attrs.append((attr > marker_values[i]))
+            else:
+                raise ValueError(_("Unknown sort direction, "
+                                   "must be 'desc' or 'asc'"))
+
+            criteria = sqlalchemy.sql.and_(*crit_attrs)
+            criteria_list.append(criteria)
+
+        f = sqlalchemy.sql.or_(*criteria_list)
+        query = query.filter(f)
+
+    if limit is not None:
+        query = query.limit(limit)
+
+    if offset:
+        query = query.offset(offset)
+
+    return query

dolphin/db/api.py

@@ -262,3 +262,8 @@ def access_info_get_all(context, marker=None, limit=None, sort_keys=None,
     """
     return IMPL.access_info_get_all(context, marker, limit, sort_keys, sort_dirs,
                                     filters, offset)
+
+
+def is_orm_value(obj):
+    """Check if object is an ORM field."""
+    return IMPL.is_orm_value(obj)