Merge branch 'dev'

api/controllers/actors/abstract_controller.py

@@ -24,6 +24,7 @@ def __init__(self):
 
         self.rmq_credentials = pika.PlainCredentials(self.rmq_user,
                                                      self.rmq_pass)
+        self.consumer = None
 
         try:
             self.redis = redis.Redis(connection_pool=settings.REDIS_CON_POOL)

api/controllers/urls.py

@@ -7,6 +7,12 @@
     url(r'^$', views.controller_list),
     url(r'^data/?$', views.ControllerData.as_view()),
     url(r'^(?P<controller_id>\w+)/data/?$', views.ControllerData.as_view()),
-    url(r'^(?P<controller_id>\d+)/?$', views.controller_detail)
+    url(r'^(?P<controller_id>\d+)/?$', views.controller_detail),
+
+    url(r'^instances/?$', views.instences_list),
+    url(r'^instance/?$', views.create_instance),
+    url(r'^instance/(?P<instance_id>\w+)/?$', views.instance_detail),
+
+
 
 ]

api/controllers/views.py

@@ -20,7 +20,7 @@
 
 
 #
-# Global Controllers
+# Controllers
 #
 @csrf_exempt
 def controller_list(request):
@@ -59,17 +59,15 @@ def controller_detail(request, controller_id):
         to_json_bools(controller, 'enabled')
         return JSONResponse(controller, status=status.HTTP_200_OK)
 
-    elif request.method == 'PUT':
-        data = JSONParser().parse(request)
+    elif request.method == 'POST':
+        print dir(request)
+        data = json.loads(request.POST['metadata'])
         try:
-            controller_data = r.hgetall('controller:' + str(controller_id))
-            to_json_bools(data, 'enabled')
-            if data['enabled']:
-                controller_name = controller_data['controller_name'].split('.')[0]
-                controller_class_name = controller_data['class_name']
-                start_controller(str(controller_id), controller_name, controller_class_name)
-            else:
-                stop_controller(str(controller_id))
+            if request.FILES['file']:
+                file_obj = request.FILES['file']
+                make_sure_path_exists(settings.CONTROLLERS_DIR)
+                path = save_file(file_obj, settings.CONTROLLERS_DIR)
+                data['controller_name'] = os.path.basename(path)
 
             r.hmset('controller:' + str(controller_id), data)
             return JSONResponse("Data updated", status=status.HTTP_201_CREATED)
@@ -79,7 +77,6 @@ def controller_detail(request, controller_id):
             return JSONResponse("Error starting controller", status=status.HTTP_400_BAD_REQUEST)
 
     elif request.method == 'DELETE':
-        stop_controller(controller_id)
         try:
             controller = r.hgetall('controller:' + str(controller_id))
             delete_file(controller['controller_name'], settings.CONTROLLERS_DIR)
@@ -125,10 +122,6 @@ def post(self, request):
 
             r.hmset('controller:' + str(controller_id), data)
 
-            if data['enabled']:
-                controller_name = data['controller_name'].split('.')[0]
-                start_controller(str(controller_id), controller_name, data['class_name'])
-
             return JSONResponse(data, status=status.HTTP_201_CREATED)
 
         except DataError:
@@ -164,26 +157,132 @@ def get(self, request, controller_id):
             return HttpResponse(status=status.HTTP_404_NOT_FOUND)
 
 
-def start_controller(controller_id, controller_name, controller_class_name):
+#
+# Instances
+#
+@csrf_exempt
+def instences_list(request):
+    """
+    List all global controllers.
+    """
+    try:
+        r = get_redis_connection()
+    except RedisError:
+        return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    if request.method == 'GET':
+        keys = r.keys('controller_instance:*')
+        controller_list = []
+        for key in keys:
+            controller = r.hgetall(key)
+            controller['id'] = key.split(':')[1]
+            controller['controller'] = r.hgetall('controller:' + controller['controller'])['controller_name']
+            controller_list.append(controller)
+        return JSONResponse(controller_list, status=status.HTTP_200_OK)
+
+    return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
+
+
+@csrf_exempt
+def create_instance(request):
+
+    try:
+        r = get_redis_connection()
+    except RedisError:
+        return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    if request.method == 'POST':
+        data = JSONParser().parse(request)
+        try:
+            r.hincrby('controller:' + data['controller'], 'instances', 1)
+            r.hmset('controller_instance:' + str(r.incr('controller_instances:id')), data)
+            return JSONResponse("Instance created", status=status.HTTP_201_CREATED)
+        except Exception:
+            return JSONResponse("Error creating instance", status=status.HTTP_400_BAD_REQUEST)
+
+    return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
+
+
+@csrf_exempt
+def instance_detail(request, instance_id):
+
+    try:
+        r = get_redis_connection()
+    except RedisError:
+        return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    if request.method == 'GET':
+        try:
+            controller = r.hgetall('controller_instance:' + str(instance_id))
+            return JSONResponse(controller, status=status.HTTP_200_OK)
+        except Exception:
+            return JSONResponse("Error retrieving data", status=status.HTTP_400_BAD_REQUEST)
+
+    elif request.method == 'PUT':
+        data = JSONParser().parse(request)
+        try:
+            if 'status' in data and data['status'] == 'Running':
+                instance_data = r.hgetall('controller_instance:' + str(instance_id))
+                controller_data = r.hgetall('controller:' + str(instance_data['controller']))
+                controller_name = controller_data['controller_name'].split('.')[0]
+                class_name = controller_data['class_name']
+                parameters = instance_data['parameters']
+
+                start_controller_instance(instance_id, controller_name, class_name, parameters)
+            else:
+                stop_controller_instance(instance_id)
+
+            r.hmset('controller_instance:' + str(instance_id), data)
+            return JSONResponse("Data updated", status=status.HTTP_201_CREATED)
+        except DataError:
+            return JSONResponse("Error updating data", status=status.HTTP_400_BAD_REQUEST)
+
+    elif request.method == 'DELETE':
+        try:
+            controller_id = 'controller:' + r.hgetall('controller_instance:' + instance_id)['controller']
+            r.hincrby(controller_id, 'instances', -1)
+            r.delete("controller_instance:" + str(instance_id))
+        except:
+            return JSONResponse("Error deleting controller", status=status.HTTP_400_BAD_REQUEST)
+
+        # If this is the last controller, the counter is reset
+        keys = r.keys('controller_instance:*')
+        if not keys:
+            r.delete('controller_instances:id')
+
+        return JSONResponse('Instance has been deleted', status=status.HTTP_204_NO_CONTENT)
+
+    return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
+
+
+def start_controller_instance(instance_id, controller_name, controller_class_name, parameters):
     host = create_local_host()
 
     controller_location = os.path.join(controller_name, controller_class_name)
+    parameters = parameters.strip().split(',')
+    # params = {}
+    params = list()
+    for parameter in parameters:
+        param_name, value = parameter.split('=')
+        # params[param_name] = value
+        params.append(value)
+
     try:
-        if controller_id not in controller_actors:
-            controller_actors[controller_id] = host.spawn(controller_name, controller_location)
-            controller_actors[controller_id].run()
+        if instance_id not in controller_actors:
+            controller_actors[instance_id] = host.spawn(controller_name, controller_location, params)
+            controller_actors[instance_id].run()
             logger.info("Controller, Started controller actor: "+controller_location)
     except Exception as e:
-        logger.error(str(e))
+        logger.error(str(e.message))
         raise ValueError
 
 
-def stop_controller(controller_id):
-    if controller_id in controller_actors:
+def stop_controller_instance(instance_id):
+    if instance_id in controller_actors:
         try:
-            controller_actors[controller_id].stop_actor()
-            del controller_actors[controller_id]
-            logger.info("Controller, Stopped controller actor: " + str(controller_id))
+            controller_actors[instance_id].stop_actor()
+            del controller_actors[instance_id]
+            logger.info("Controller, Stopped controller actor: " + str(instance_id))
         except Exception as e:
             logger.error(str(e))
             raise ValueError

api/projects/views.py

@@ -54,13 +54,16 @@ def projects(request, project_id=None):
             keystone_client.roles.grant(role=admin_role_id, user=admin_user_id, project=project_id)
 
             # Post Storlet and Dependency containers
-            admin_user = settings.MANAGEMENT_ADMIN_USERNAME
-            headers = {'X-Container-Read': '*:' + admin_user, 'X-Container-Write': '*:' + admin_user}
             url, token = get_swift_url_and_token(project_name)
-            swift_client.put_container(url, token, "storlet", headers)
-            swift_client.put_container(url, token, "dependency", headers)
+            try:
+                swift_client.put_container(url, token, "storlet")
+                swift_client.put_container(url, token, "dependency")
+                headers = {'X-Account-Meta-Crystal-Enabled': True, 'X-Account-Meta-Storlet-Enabled': True}
+                swift_client.post_account(url, token, headers)
+            except:
+                pass
             # Create project docker image
-            create_docker_image(project_id)
+            create_docker_image(r, project_id)
 
             r.lpush('projects_crystal_enabled', project_id)
             return JSONResponse("Data inserted correctly", status=status.HTTP_201_CREATED)
@@ -74,16 +77,20 @@ def projects(request, project_id=None):
 
             # Delete Storlet and Dependency containers
             url, token = get_swift_url_and_token(project_name)
-            swift_client.delete_container(url, token, "storlet")
-            swift_client.delete_container(url, token, "dependency")
-
+            try:
+                swift_client.delete_container(url, token, "storlet")
+                swift_client.delete_container(url, token, "dependency")
+                headers = {'X-Account-Meta-Crystal-Enabled': '', 'X-Account-Meta-Storlet-Enabled': ''}
+                swift_client.post_account(url, token, headers)
+            except:
+                pass
             # Delete Manager as admin of the Crystal Project
             keystone_client = get_keystone_admin_auth()
             admin_role_id, admin_user_id = get_admin_role_user_ids()
             keystone_client.roles.revoke(role=admin_role_id, user=admin_user_id, project=project_id)
 
             # Delete project docker image
-            delete_docker_image(project_id)
+            delete_docker_image(r, project_id)
 
             r.lrem('projects_crystal_enabled', project_id)
             return JSONResponse("Crystal project correctly disabled.", status=status.HTTP_201_CREATED)
@@ -103,12 +110,16 @@ def projects(request, project_id=None):
     return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
 
 
-def create_docker_image(project_id):
-    pass
+def create_docker_image(r, project_id):
+    nodes = r.keys('*_node:*')
+    for node in nodes:
+        node_data = r.hgetall(node)
 
 
-def delete_docker_image(project_id):
-    pass
+def delete_docker_image(r, project_id):
+    nodes = r.keys('*_node:*')
+    for node in nodes:
+        node_data = r.hgetall(node)
 
 
 #

api/swift/urls.py

@@ -4,6 +4,7 @@
 urlpatterns = [
     # Storages Policies
     url(r'^storage_policies/?$', views.storage_policies),
+    url(r'^storage_policy/(?P<storage_policy_id>[^/]+)/?$', views.storage_policy_detail),
 
     # Object Placement
     url(r'^locality/(?P<account>\w+)(?:/(?P<container>[-\w]+))(?:/(?P<swift_object>[-\w]+))?/$', views.locality_list),
@@ -21,5 +22,5 @@
     url(r'^regions/(?P<region_id>[^/]+)/?$', views.region_detail),
     url(r'^zones/?$', views.zones),
     url(r'^zones/(?P<zone_id>[^/]+)/?$', views.zone_detail),
-
+    
 ]

api/swift/views.py

@@ -46,11 +46,11 @@ def storage_policies(request):
         data = JSONParser().parse(request)
         storage_nodes_list = []
         if isinstance(data["storage_node"], dict):
+            data['storage_node']['policy_id'] = r.incr('storage-policies:id')
             for k, v in data["storage_node"].items():
                 storage_nodes_list.extend([k, v])
             data["storage_node"] = ','.join(map(str, storage_nodes_list))
             try:
-                print data
                 storage_policies_utils.create(data)
             except Exception as e:
                 return JSONResponse('Error creating the Storage Policy: ' + e, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -60,6 +60,37 @@ def storage_policies(request):
     return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
 
 
+@csrf_exempt
+def storage_policy_detail(request, storage_policy_id):
+
+    try:
+        r = get_redis_connection()
+    except RedisError:
+        return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    key = "storage-policy:" + storage_policy_id
+    if request.method == 'GET':
+        if r.exists(key):
+            storage_policy = r.hgetall(key)
+            storage_policy['devices'] = json.loads(storage_policy['devices'])
+            return JSONResponse(storage_policy, status=status.HTTP_200_OK)
+        else:
+            return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND)
+
+    if request.method == 'PUT':
+        if r.exists(key):
+            data = JSONParser().parse(request)
+            try:
+                r.hmset(key, data)
+                return JSONResponse("Storage Policy updated", status=status.HTTP_201_CREATED)
+            except RedisError:
+                return JSONResponse("Error updating storage policy", status=status.HTTP_400_BAD_REQUEST)
+        else:
+            return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND)
+
+    return JSONResponse('Method not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
+
+
 @csrf_exempt
 def locality_list(request, account, container=None, swift_object=None):
     """
@@ -81,8 +112,6 @@ def locality_list(request, account, container=None, swift_object=None):
 #
 # Node part
 #
-
-
 @csrf_exempt
 def node_list(request):
     """
@@ -365,6 +394,3 @@ def zone_detail(request, zone_id):
             return JSONResponse("Error updating zone data", status=status.HTTP_400_BAD_REQUEST)
 
     return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
-
-
-

controller_samples/static_bandwidth.py

@@ -4,9 +4,9 @@
 
 class StaticBandwidthPerProject(AbstractController):
 
-    def __init__(self):
+    def __init__(self, method):
         super(StaticBandwidthPerProject, self).__init__()
-        self.method = "get"
+        self.method = method
         self.metrics = [self.method+'_bandwidth']
         self.prev_assignations = dict()