Merge branch 'dev'

.travis.yml

@@ -1,21 +1,19 @@
 language: python
-
 python:
-  - "2.7"
-
+- '2.7'
 services:
-  - redis-server
-
-# command to install dependencies
+- redis-server
 install:
-  - pip install -r requirements.txt
-  - pip install -r test-requirements.txt
-  - git clone https://github.com/cloudspaces/pyactive.git && cd pyactive/pyactive_project && python setup.py develop && cd ../..
-  #- pip install -q $DJANGO --use-mirrors
-
-# command to run tests
+- pip install -r requirements.txt
+- pip install -r test-requirements.txt
+- git clone https://github.com/cloudspaces/pyactive.git && cd pyactive/pyactive_project
+  && python setup.py develop && cd ../..
 script:
-  - cd sds_controller/ && coverage run --source='.' manage.py test
-#  - coverage run --source='.' manage.py test
-after_success:
-  coveralls
+- cd sds_controller/ && coverage run --source='.' manage.py test
+after_success: coveralls
+notifications:
+  slack:
+    rooms:
+      secure: LA5rDP5xhGjLQma0VqoYKc+VUOf6Iwhffe4D7kqfC07gm29RnRkfX+B5WeIYGKln5CKVFeKVoSTVoPuOwKrylZEfSwjnrqVIF6nzPGSXaq7iC/DL5P9JyHOWmaTXQXSkfbOTuTwQ8fB6yS+umI0ZhtWS7L+haDW55VEs4MaRzeuei1wpcQARt1x/OhBPW/551PP8R3Qd31bk5acCYlsiO4tMeDJLmiqAc+pZCPP+LfM2MdaM//qSwGF7cmrl75Pzs1nzHtF3pKX1SDCQCZ7cjXpGUG3J51Nak1Oyfu96Jr/AfPSc3NgwNdlGL+LAIszn9bjS9BQAHEYIL/hmJPs34pasdN+bSAE9nxVO81c9ISsJxH/ZOHw2+4jgx+ylz7PLKl6wcyBrxvuFmRGm/jiPAlmhQ99MUoiQ9OTnJKUzp2VhI4GwjMwJ3Zhgh61JoYaQmaY1Uwa+VNp1Xu4/JQumcOlkS8WZghArj8Xh2KiJShSHCJw4gOKHaA/uUNRzlCeJOEg0SUdlLdM9Wd7a001ktl6XgIIDamXkUrTLA0uNYS9bMHURzGAekvTFNSyqPJVQm66c1kKT1km5kIqFEDW+f4Jpr9SgyW3pGJ21HBCv64JHl3TtH1bImu4tzW/5Velid/le1sxRDLrf31l4wOTLfXg5Qym9l4r0Nr8CWFnYEd4=
+    on_success: change
+    on_failure: always

dynamic_policies/docs/conf.py

@@ -14,7 +14,7 @@
 
 import sys
 import os
-import shlex
+# import shlex
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the

dynamic_policies/dsl_parser.py

@@ -1,7 +1,6 @@
 from pyparsing import Word, Group, Literal, nums, alphanums, operatorPrecedence, Regex, oneOf
 from pyparsing import opAssoc, Suppress, alphas, Combine, delimitedList, ZeroOrMore, Optional
 import redis
-from django.conf import settings
 
 # By default, PyParsing treats \n as whitespace and ignores it
 # In our grammer, \n is significant, so tell PyParsing not to ignore it
@@ -21,8 +20,7 @@
 
 
 def get_redis_connection():
-    return redis.Redis(connection_pool=settings.REDIS_CON_POOL)
-
+    return redis.StrictRedis(host="localhost", port=6379, db=0)
 
 def parse_group_tenants(tokens):
     r = get_redis_connection()
@@ -71,7 +69,7 @@ def parse(input_string):
 
     # Action part
     action = oneOf("SET DELETE")
-    sfilters_list = r.keys("filter:*")
+    sfilters_list = r.keys("dsl_filter:*")
     sfilter = map(lambda x: "".join(x.split(":")[1]), sfilters_list)
 
     with_params = Suppress(Literal("WITH"))
@@ -117,7 +115,7 @@ def parse(input_string):
 
     for action in parsed_rule.action_list:
         if action.params:
-            filter_info = r.hgetall("filter:"+str(action.filter))
+            filter_info = r.hgetall("dsl_filter:"+str(action.filter))
             if "valid_parameters" in filter_info.keys():
                 params = eval(filter_info["valid_parameters"])
                 result = set(action.params.keys()).intersection(params.keys())

dynamic_policies/dynamic_policies.config

@@ -11,10 +11,10 @@ port=6379
 db=0
 
 [openstack]
-admin_name=swift
-admin_pass=urv
-admin_tenant=service
-keystone_url=http://10.30.235.235:5000/v2.0/tokens
+admin_name=storlet_manager
+admin_pass=storlet_manager
+admin_tenant=storlet_management
+keystone_url=http://localhost:5000/v2.0/tokens
 
 [logstash]
 host=iostack.urv.cat

dynamic_policies/init_actors.py

@@ -86,6 +86,8 @@ def start_redis_rules(host, rules):
     r = get_redis_connection()
     dynamic_policies = r.keys("policy:*")
 
+
+
     if dynamic_policies:
         print "\nStarting dynamic rules stored in redis:"
 
@@ -94,9 +96,7 @@ def start_redis_rules(host, rules):
 
         if policy_data['alive'] == 'True':
             _, rule_parsed = dsl_parser.parse(policy_data['policy']) 
-
             target = rule_parsed.target[0][1]  # Tenant ID or tenant+container
-
             for action_info in rule_parsed.action_list:
                 if action_info.transient:
                     print 'Transient rule:', policy_data['policy']

dynamic_policies/metrics/swift_metric.py

@@ -29,15 +29,17 @@ def notify(self, body):
 
         data = json.loads(body)
         Thread(target=self._send_data_to_logstash, args=(data, )).start()
-
-        """
+
         try:
-            for observer in self._observers[body_parsed.target]:
-                observer.update(self.name, body_parsed)
+            for host in data:
+                for target in data[host]:
+                    value =  data[host][target]
+                    target = target.replace('AUTH_','')
+                    if target in self._observers:
+                        for observer in self._observers[target]:
+                            observer.update(self.name, value)
         except:
-            #print "fail", body_parsed
-            pass
-        """
+            print "Fail sending to observer: ", data       
 
     def get_value(self):
         return self.value
@@ -70,4 +72,3 @@ def _send_data_to_logstash(self, data):
                 monitoring_data = dict()
         except socket.error:
             print "Error sending monitoring data to logstash"
-            pass

dynamic_policies/rules/rule.py

@@ -75,6 +75,7 @@ def admin_login(self):
         body = json.dumps({"auth": {"tenantName": self.openstack_tenant, "passwordCredentials": {"username": self.openstack_user,
                                                                                                  "password": self.openstack_pass}}})
         headers = {"Content-type": "application/json"}
+
         r = requests.post(self.openstack_keystone_url, data=body, headers=headers)
         if r.status_code == 200:
             self.token = r.json()["access"]["token"]["id"]
@@ -137,7 +138,7 @@ def check_metrics(self, condition_list):
                 if element is not "OR" and element is not "AND":
                     self.check_metrics(element)
 
-    def update(self, metric, tenant_info):
+    def update(self, metric, value):
         """
         The method update is called by the workloads metrics following the observer
         pattern. This method is called to send to this actor the data updated.
@@ -148,9 +149,11 @@ def update(self, metric, tenant_info):
         :param tenant_info: Contains the timestamp and the value sent from workload metric.
         :type tenant_info: **any** PyParsing type
         """
-        print 'Success update:  ', tenant_info
+        print 'Success update:  ', value
 
-        self.observers_values[metric] = tenant_info.value
+        self.observers_values[metric] = value
+
+        print self.observers_values
 
         # TODO Check the last time updated the value
         # Check the condition of the policy if all values are setted. If the condition
@@ -196,11 +199,13 @@ def do_action(self):
         is responsible to execute the action defined in the policy.
 
         """
+
         if not self.token:
             self.admin_login()
 
         headers = {"X-Auth-Token": self.token}
-        dynamic_filter = self.redis.hgetall("filter:"+str(self.action_list.filter))
+
+        dynamic_filter = self.redis.hgetall("dsl_filter:"+str(self.action_list.filter))      
 
         if self.action_list.action == "SET":
             print "--> SET <--"

dynamic_policies/rules/rule_transient.py

@@ -67,7 +67,7 @@ def do_action(self, condition_result):
             self.admin_login()
 
         headers = {"X-Auth-Token": self.token}
-        dynamic_filter = self.redis.hgetall("filter:"+str(self.action_list.filter))
+        dynamic_filter = self.redis.hgetall("dsl_filter:"+str(self.action_list.filter))
 
         if action == "SET":
 

sds_controller/bw/tests.py

@@ -1,7 +1,10 @@
+import json
+import mock
 import redis
 
-from django.test import TestCase, override_settings
 from django.conf import settings
+from django.http import HttpResponse
+from django.test import TestCase, override_settings
 from rest_framework import status
 from rest_framework.test import APIRequestFactory
 
@@ -15,10 +18,12 @@ def setUp(self):
         # Every test needs access to the request factory.
         # Using rest_framework's APIRequestFactory: http://www.django-rest-framework.org/api-guide/testing/
         self.factory = APIRequestFactory()
+        self.r = redis.Redis(connection_pool=settings.REDIS_CON_POOL)
+        self.create_storage_policies()
+        self.create_bw_policies()
 
     def tearDown(self):
-        r = redis.Redis(connection_pool=settings.REDIS_CON_POOL)
-        r.flushdb()
+        self.r.flushdb()
 
     def test_bw_list_with_method_not_allowed(self):
         """ Test that DELETE requests to bw_list() return METHOD_NOT_ALLOWED """
@@ -28,9 +33,9 @@ def test_bw_list_with_method_not_allowed(self):
 
     def test_bw_detail_with_method_not_allowed(self):
         """ Test that POST requests to bw_detail() return METHOD_NOT_ALLOWED """
-        tenant_key = '123456789abcdef:1'
-        request = self.factory.post('/bw/sla/' + tenant_key)
-        response = bw_detail(request, tenant_key)
+        project_policy_key = '123456789abcdef:1'
+        request = self.factory.post('/bw/sla/' + project_policy_key)
+        response = bw_detail(request, project_policy_key)
         self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
 
     def test_get_slas_without_auth_token(self):
@@ -41,7 +46,69 @@ def test_get_slas_without_auth_token(self):
 
     def test_get_sla_detail_without_auth_token(self):
         # Create an instance of a GET request without auth token
-        tenant_key = '123456789abcdef:1'
-        request = self.factory.get('/bw/sla/' + tenant_key)
-        response = bw_detail(request, tenant_key)
+        project_policy_key = '123456789abcdef:1'
+        request = self.factory.get('/bw/sla/' + project_policy_key)
+        response = bw_detail(request, project_policy_key)
         self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
+
+    @mock.patch('bw.views.requests.get')
+    def test_bw_list_ok(self, mock_requests_get):
+        """ Test that a GET request to bw_list() returns OK """
+
+        resp = HttpResponse()
+        resp.content = json.dumps({'tenants': [{'name': 'tenantA', 'id': '0123456789abcdef'},
+                                               {'name': 'tenantB', 'id': 'abcdef0123456789'}]})
+        mock_requests_get.return_value = resp
+
+        request = self.factory.get('/bw/slas')
+        request.META['HTTP_X_AUTH_TOKEN'] = 'fake_token'
+        response = bw_list(request)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        json_data = json.loads(response.content)
+        self.assertEqual(len(json_data), 2)
+        sorted_data = sorted(json_data, key=lambda datum: datum['policy_id'])
+        self.assertEqual(sorted_data[0]['policy_id'], '2')
+        self.assertEqual(sorted_data[0]['project_id'], '0123456789abcdef')
+        self.assertEqual(sorted_data[0]['bandwidth'], '2000')
+        self.assertEqual(sorted_data[0]['project_name'], 'tenantA')
+        self.assertEqual(sorted_data[0]['policy_name'], 's0y1')
+        self.assertEqual(sorted_data[1]['policy_id'], '3')
+        self.assertEqual(sorted_data[1]['project_id'], 'abcdef0123456789')
+        self.assertEqual(sorted_data[1]['bandwidth'], '3000')
+        self.assertEqual(sorted_data[1]['project_name'], 'tenantB')
+        self.assertEqual(sorted_data[1]['policy_name'], 's3y4')
+
+    @mock.patch('bw.views.requests.get')
+    def test_bw_detail_ok(self, mock_requests_get):
+        """ Test that a GET request to bw_list() returns OK """
+
+        resp = HttpResponse()
+        resp.content = json.dumps({'tenants': [{'name': 'tenantA', 'id': '0123456789abcdef'},
+                                               {'name': 'tenantB', 'id': 'abcdef0123456789'}]})
+        mock_requests_get.return_value = resp
+
+        project_policy_key = '0123456789abcdef:2'
+        request = self.factory.get('/bw/slas/' + project_policy_key)
+        request.META['HTTP_X_AUTH_TOKEN'] = 'fake_token'
+        response = bw_detail(request, project_policy_key)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        json_data = json.loads(response.content)
+        self.assertEqual(json_data['policy_id'], '2')
+        self.assertEqual(json_data['project_id'], '0123456789abcdef')
+        self.assertEqual(json_data['bandwidth'], '2000')
+        self.assertEqual(json_data['project_name'], 'tenantA')
+
+    #
+    # Aux functions
+    #
+
+    def create_storage_policies(self):
+        self.r.hmset("storage-policy:0", {'name': 'allnodes', 'default': 'yes', 'policy_type': 'replication'})
+        self.r.hmset("storage-policy:1", {'name': 'storage4', 'default': 'no', 'policy_type': 'replication'})
+        self.r.hmset("storage-policy:2", {'name': 's0y1', 'default': 'no', 'policy_type': 'replication'})
+        self.r.hmset("storage-policy:3", {'name': 's3y4', 'default': 'no', 'policy_type': 'replication'})
+        self.r.hmset("storage-policy:4", {'name': 's5y6', 'default': 'no', 'policy_type': 'replication'})
+
+    def create_bw_policies(self):
+        self.r.hmset('bw:AUTH_0123456789abcdef', {'2': '2000'})
+        self.r.hmset('bw:AUTH_abcdef0123456789', {'3': '3000'})

sds_controller/bw/urls.py

@@ -4,5 +4,5 @@
 
 urlpatterns = [
     url(r'^/slas/?$', views.bw_list),
-    url(r'^/sla/(?P<tenant_key>[^/]+)/?$', views.bw_detail),
+    url(r'^/sla/(?P<project_key>[^/]+)/?$', views.bw_detail),
 ]