Changed filter data, deleted DSL mapping

api/api/settings.py

@@ -23,11 +23,10 @@
 DEPENDENCY_DIR = os.path.join('/opt', 'crystal', 'dependencies')
 CONTROLLERS_DIR = os.path.join('/opt', 'crystal', 'controllers')
 
-NATIVE_FILTER_KEYS = ('id', 'filter_name', 'filter_type', 'language', 'dependencies', 'main', 'is_pre_put', 'is_post_put',
-                      'is_pre_get', 'is_post_get', 'has_reverse', 'execution_server', 'execution_server_reverse', 'path')
-STORLET_FILTER_KEYS = ('id', 'filter_name', 'filter_type', 'language', 'interface_version', 'dependencies', 'main', 'is_pre_put', 'is_post_put',
-                       'is_pre_get', 'is_post_get', 'has_reverse', 'execution_server', 'execution_server_reverse', 'path')
+NATIVE_FILTER_KEYS = ('id', 'filter_name', 'filter_type', 'language', 'dsl_name', 'dependencies', 'main', 'put', 'get', 'execution_server', 'reverse', 'path')
+STORLET_FILTER_KEYS = ('id', 'filter_name', 'filter_type', 'language', 'dsl_name', 'interface_version', 'dependencies', 'main', 'put', 'get', 'execution_server', 'reverse', 'path')
 DEPENDENCY_KEYS = ('id', 'name', 'version', 'permissions', 'path')
+
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
 

api/filters/urls.py

@@ -4,18 +4,18 @@
 urlpatterns = [
 
     url(r'^$', views.filter_list),
-    url(r'^(?P<filter_id>[0-9]+)/?$', views.filter_detail),
-    url(r'^(?P<filter_id>[0-9]+)/data/?$', views.FilterData.as_view()),
+    url(r'^(?P<filter_id>\w+)/?$', views.filter_detail),
+    url(r'^(?P<filter_id>\w+)/data/?$', views.FilterData.as_view()),
 
     # Deploy to project container or object
-    url(r'^(?P<project_id>\w+)/deploy/(?P<filter_id>[0-9]+)/?$', views.filter_deploy),
-    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/deploy/(?P<filter_id>[0-9]+)/?$', views.filter_deploy),
-    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/(?P<swift_object>[-\w]+)/deploy/(?P<filter_id>[0-9]+)/?$', views.filter_deploy),
+    url(r'^(?P<project_id>\w+)/deploy/(?P<filter_id>\w+)/?$', views.filter_deploy),
+    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/deploy/(?P<filter_id>\w+)/?$', views.filter_deploy),
+    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/(?P<swift_object>[-\w]+)/deploy/(?P<filter_id>\w+)/?$', views.filter_deploy),
 
     # Undeploy to project container or object
-    url(r'^(?P<project_id>\w+)/undeploy/(?P<filter_id>[0-9]+)/?$', views.filter_undeploy),
-    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/undeploy/(?P<filter_id>[0-9]+)/?$', views.filter_undeploy),
-    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/(?P<swift_object>[-\w]+)/undeploy/(?P<filter_id>[0-9]+)/?$', views.filter_undeploy),
+    url(r'^(?P<project_id>\w+)/undeploy/(?P<filter_id>\w+)/?$', views.filter_undeploy),
+    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/undeploy/(?P<filter_id>\w+)/?$', views.filter_undeploy),
+    url(r'^(?P<project_id>\w+)/(?P<container>[-\w]+)/(?P<swift_object>[-\w]+)/undeploy/(?P<filter_id>\w+)/?$', views.filter_undeploy),
 
     url(r'^dependencies/?$', views.dependency_list),
     url(r'^dependencies/(?P<dependency_id>\w+)/?$', views.dependency_detail),
@@ -24,8 +24,4 @@
     url(r'^dependencies/(?P<project_id>\w+)/deploy/(?P<dependency_id>\w+)/?$', views.dependency_deploy),
     url(r'^dependencies/(?P<project_id>\w+)/undeploy/(?P<dependency_id>\w+)/?$', views.dependency_undeploy),
 
-    # Filters DSL Mapping
-    url(r'^dsl/?$', views.add_dsl_filter),
-    url(r'^dsl/(?P<name>\w+)/?$', views.dsl_filter_detail),
-
 ]

api/filters/views.py

@@ -17,7 +17,7 @@
 import mimetypes
 import os
 
-from api.common import rsync_dir_with_nodes, to_json_bools, JSONResponse, \
+from api.common import rsync_dir_with_nodes, JSONResponse, \
     get_redis_connection, get_token_connection, make_sure_path_exists, save_file, md5
 from api.exceptions import SwiftClientError, StorletNotFoundException, FileSynchronizationException
 
@@ -58,10 +58,10 @@ def filter_list(request):
            (data['filter_type'] == 'storlet' and not check_keys(data.keys(), settings.STORLET_FILTER_KEYS[2:-1]))):
             return JSONResponse("Invalid parameters in request", status=status.HTTP_400_BAD_REQUEST)
 
-        storlet_id = r.incr("filters:id")
         try:
-            data['id'] = storlet_id
-            r.hmset('filter:' + str(storlet_id), data)
+            filter_id = r.incr("filters:id")
+            data['id'] = filter_id
+            r.hmset('filter:' + str(data['dsl_name']), data)
 
             return JSONResponse(data, status=status.HTTP_201_CREATED)
 
@@ -75,7 +75,6 @@ def filter_detail(request, filter_id):
     """
     Retrieve, update or delete a Filter.
     """
-
     try:
         r = get_redis_connection()
     except RedisError:
@@ -86,8 +85,6 @@ def filter_detail(request, filter_id):
 
     if request.method == 'GET':
         my_filter = r.hgetall("filter:" + str(filter_id))
-
-        to_json_bools(my_filter, 'has_reverse', 'is_pre_get', 'is_post_get', 'is_pre_put', 'is_post_put', 'enabled')
         return JSONResponse(my_filter, status=status.HTTP_200_OK)
 
     elif request.method == 'PUT':
@@ -97,21 +94,28 @@ def filter_detail(request, filter_id):
             return JSONResponse("Invalid format or empty request", status=status.HTTP_400_BAD_REQUEST)
 
         try:
-            r.hmset('filter:' + str(filter_id), data)
+            if str(filter_id) != data['dsl_name']:
+                # Check for possible activated policies
+                policies = r.keys('policy:*')
+                for policy_key in policies:
+                    policy = r.hgetall(policy_key)
+                    dsl_filter = policy['filter']
+                    if dsl_filter == str(filter_id):
+                        return JSONResponse("It is not possible to change the DSL Name, "+str(filter_id)+
+                                            " is associated with some Dynamic Policy", status=status.HTTP_400_BAD_REQUEST)
+                filter_data = r.hgetall("filter:" + str(filter_id))
+                r.hmset('filter:' + str(data['dsl_name']), filter_data)
+                r.delete("filter:" + str(filter_id))
+                r.hmset('filter:' + str(data['dsl_name']), data)
+            else:
+                r.hmset('filter:' + str(filter_id), data)
 
             return JSONResponse("Data updated", status=status.HTTP_200_OK)
         except DataError:
             return JSONResponse("Error updating data", status=status.HTTP_408_REQUEST_TIMEOUT)
 
     elif request.method == 'DELETE':
         try:
-            keys = r.keys('dsl_filter:*')
-            for key in keys:
-                dsl_filter_id = r.hget(key, 'identifier')
-                if dsl_filter_id == filter_id:
-                    return JSONResponse('Unable to delete filter, is in use by the Registry DSL.', status=status.HTTP_403_FORBIDDEN)
-
-            my_filter = r.hgetall("filter:" + str(filter_id))
             r.delete("filter:" + str(filter_id))
 
             return JSONResponse('Filter has been deleted', status=status.HTTP_204_NO_CONTENT)
@@ -467,7 +471,7 @@ def dependency_undeploy(request, dependency_id, project_id):
 def set_filter(r, target, filter_data, parameters, token):
     if filter_data['filter_type'] == 'storlet':
 
-        metadata = {"X-Object-Meta-Storlet-Language": 'java',
+        metadata = {"X-Object-Meta-Storlet-Language": filter_data["language"],
                     "X-Object-Meta-Storlet-Interface-Version": filter_data["interface_version"],
                     "X-Object-Meta-Storlet-Dependency": '',
                     "X-Object-Meta-Storlet-Object-Metadata": filter_data["object_metadata"],
@@ -535,77 +539,3 @@ def unset_filter(r, target, filter_data, token):
         json_value = json.loads(value)
         if json_value["filter_name"] == filter_data["filter_name"]:
             r.hdel("pipeline:" + str(target), key)
-
-
-#
-# DSL Mappings
-#
-@csrf_exempt
-def add_dsl_filter(request):
-    """
-    Add a filter with its default parameters in the registry (redis).
-    List all the dynamic filters registered.
-    """
-
-    try:
-        r = get_redis_connection()
-    except RedisError:
-        return JSONResponse('Error connecting with DB', status=500)
-    if request.method == 'GET':
-        keys = r.keys("dsl_filter:*")
-        dynamic_filters = []
-        for key in keys:
-            dynamic_filter = r.hgetall(key)
-            dynamic_filter["name"] = key.split(":")[1]
-            dynamic_filters.append(dynamic_filter)
-        return JSONResponse(dynamic_filters, status=200)
-
-    if request.method == 'POST':
-        data = JSONParser().parse(request)
-        name = data.pop("name", None)
-        if not name:
-            return JSONResponse('Filter must have a name', status=400)
-        r.hmset('dsl_filter:' + str(name), data)
-        return JSONResponse('Filter has been added to the registy', status=201)
-    return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=405)
-
-
-@csrf_exempt
-def dsl_filter_detail(request, name):
-    """
-    Get, update or delete a dynamic filter from the registry.
-    """
-
-    try:
-        r = get_redis_connection()
-    except RedisError:
-        return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
-    if request.method == 'GET':
-        dynamic_filter = r.hgetall("dsl_filter:" + str(name))
-        return JSONResponse(dynamic_filter, status=status.HTTP_200_OK)
-
-    if request.method == 'PUT':
-        if not r.exists('dsl_filter:' + str(name)):
-            return JSONResponse('Dynamic filter with name:  ' + str(name) + ' does not exist.', status=status.HTTP_404_NOT_FOUND)
-        data = JSONParser().parse(request)
-        if 'name' in data:
-            del data['name']
-        r.hmset('dsl_filter:' + str(name), data)
-        return JSONResponse('The metadata of the dynamic filter with name: ' + str(name) + ' has been updated',
-                            status=status.HTTP_201_CREATED)
-
-    if request.method == 'DELETE':
-        filter_id = r.hget('dsl_filter:' + str(name), 'identifier')
-        filter_name = r.hget('filter:' + str(filter_id), 'filter_name')
-
-        keys = r.keys("pipeline:*")
-        for it in keys:
-            for value in r.hgetall(it).values():
-                json_value = json.loads(value)
-                if json_value['filter_name'] == filter_name:
-                    return JSONResponse('Unable to delete Registry DSL, is in use by some policy.', status=status.HTTP_403_FORBIDDEN)
-
-        r.delete("dsl_filter:" + str(name))
-        return JSONResponse('Dynamic filter has been deleted', status=status.HTTP_204_NO_CONTENT)
-    return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)

api/policies/actors/rule.py

@@ -4,6 +4,7 @@
 import logging
 import redis
 import requests
+import os
 
 from api.settings import MANAGEMENT_ACCOUNT, MANAGEMENT_ADMIN_USERNAME, \
     MANAGEMENT_ADMIN_PASSWORD, KEYSTONE_ADMIN_URL, REDIS_HOST, REDIS_PORT, REDIS_DATABASE
@@ -25,7 +26,7 @@ class Rule(object):
     _ask = ['get_target']
     _tell = ['update', 'start_rule', 'stop_actor']
 
-    def __init__(self, rule_parsed, action, target_id, target_name):
+    def __init__(self, rule_parsed, action, target_id, target_name, controller_server):
         """
         Initialize all the variables needed for the rule.
 
@@ -56,6 +57,7 @@ def __init__(self, rule_parsed, action, target_id, target_name):
         self.action_list = action
         self.target_id = target_id
         self.target_name = target_name
+        self.controller_server = controller_server
 
         self.conditions = rule_parsed.condition_list.asList()
         self.observers_values = dict()
@@ -194,12 +196,11 @@ def _do_action(self):
             self._get_admin_token()
 
         headers = {"X-Auth-Token": self.token}
-        dynamic_filter = self.redis.hgetall("dsl_filter:" + str(self.action_list.filter))
 
         if self.action_list.action == "SET":
             # TODO Review if this tenant has already deployed this filter. Not deploy the same filter more than one time.
 
-            url = dynamic_filter["activation_url"] + "/" + self.target_id + "/deploy/" + str(dynamic_filter["identifier"])
+            url = os.path.join(self.controller_server, 'filters', self.target_id, "deploy", str(self.action_list.filter))
 
             data = dict()
 
@@ -229,9 +230,8 @@ def _do_action(self):
                 logger.error('Error setting policy')
 
         elif self.action_list.action == "DELETE":
-            logger.info("--> DELETE <--")
 
-            url = dynamic_filter["activation_url"] + "/" + self.target_id + "/undeploy/" + str(dynamic_filter["identifier"])
+            url = os.path.join(self.controller_server, 'filters', self.target_id, "undeploy", str(self.action_list.filter))
             response = requests.put(url, headers=headers)
 
             if 200 <= response.status_code < 300:

api/policies/actors/rule_transient.py

@@ -2,6 +2,7 @@
 import requests
 import logging
 import json
+import os
 
 logger = logging.getLogger(__name__)
 
@@ -19,7 +20,7 @@ class TransientRule(Rule):
     _ask = ['get_target']
     _async = ['update', 'start_rule', 'stop_actor']
 
-    def __init__(self, rule_parsed, action, target_id, target_name):
+    def __init__(self, rule_parsed, action, target_id, target_name, controller_server):
         """
         Initialize all the variables needed for the rule.
 
@@ -28,7 +29,8 @@ def __init__(self, rule_parsed, action, target_id, target_name):
         :param target_name: The target assigned to this rule.
         :type target_name: **any** String type
         """
-        super(TransientRule, self).__init__(rule_parsed, action, target_id, target_name)
+        super(TransientRule, self).__init__(rule_parsed, action, target_id,
+                                            target_name, controller_server)
         logger.info("Transient Rule")
         self.execution_stat = False
         self.static_policy_id = None
@@ -72,13 +74,12 @@ def do_action(self, condition_result):
             self._get_admin_token()
 
         headers = {"X-Auth-Token": self.token}
-        dynamic_filter = self.redis.hgetall("dsl_filter:"+str(self.action_list.filter))
 
         if action == "SET":
             # TODO Review if this tenant has already deployed this filter. Don't deploy the same filter more than one time.
             logger.info("Setting static policy")
             data = dict()
-            url = dynamic_filter["activation_url"]+"/"+self.target_id+"/deploy/"+str(dynamic_filter["identifier"])
+            url = os.path.join(self.controller_server, 'filters', self.target_id, "deploy", str(self.action_list.filter))
 
             if hasattr(self.rule_parsed.object_list, "object_type"):
                 data['object_type'] = self.rule_parsed.object_list.object_type.object_value
@@ -102,8 +103,7 @@ def do_action(self, condition_result):
 
         elif action == "DELETE":
             logger.info("Deleting static policy " + str(self.static_policy_id))
-
-            url = dynamic_filter["activation_url"].rsplit("/",1)[0]+"/controller/static_policy/"+self.target_id+":"+str(self.static_policy_id)
+            url = os.path.join(self.controller_server, "policies/static", self.target_id+":"+str(self.static_policy_id))
             response = requests.delete(url, headers=headers)
 
             if 200 <= response.status_code < 300:

api/policies/dsl_parser.py

@@ -67,7 +67,7 @@ def parse(input_string):
     # Group(tenant_list ^ tenant_group_list ^ container_list ^ obj_list)
     # Action part
     action = oneOf("SET DELETE")
-    sfilters_list = r.keys("dsl_filter:*")
+    sfilters_list = r.keys("filter:*")
     sfilter = map(lambda x: "".join(x.split(":")[1]), sfilters_list)
     with_params = Suppress(Literal("WITH"))
     do = Suppress(Literal("DO"))