Merge branch 'dev'

api/api/common_utils.py

@@ -44,7 +44,7 @@ def get_keystone_admin_auth():
 def is_valid_request(request):    
     token = request.META['HTTP_X_AUTH_TOKEN']
     is_admin = False
-    now = datetime.now()
+    now = datetime.utcnow()
 
     if token not in valid_tokens:
         keystone = get_keystone_admin_auth()
@@ -53,7 +53,7 @@ def is_valid_request(request):
             token_data = keystone.tokens.validate(token)
         except:
             return False
-          
+
         token_expiration = datetime.strptime(token_data.expires, 
                                              '%Y-%m-%dT%H:%M:%SZ')
 
@@ -70,6 +70,8 @@ def is_valid_request(request):
         token_expiration = valid_tokens[token]
         if token_expiration > now:
             return token
+        else:
+            valid_tokens.pop(token, None)
 
     return False
 

api/api/tests.py

@@ -4,16 +4,21 @@
 from django.conf import settings
 from django.core.urlresolvers import resolve
 from django.test import TestCase, override_settings
+from datetime import datetime, timedelta
+from rest_framework.test import APIRequestFactory
 
-from .common_utils import get_all_registered_nodes, remove_extra_whitespaces, to_json_bools, rsync_dir_with_nodes
+from .common_utils import get_all_registered_nodes, remove_extra_whitespaces, to_json_bools, rsync_dir_with_nodes, is_valid_request, get_project_list, \
+    get_keystone_admin_auth
 from .exceptions import FileSynchronizationException
+from .startup import run as startup_run
 
 # Tests use database=10 instead of 0.
 @override_settings(REDIS_CON_POOL=redis.ConnectionPool(host='localhost', port=6379, db=10))
 class MainTestCase(TestCase):
     def setUp(self):
         self.r = redis.Redis(connection_pool=settings.REDIS_CON_POOL)
         self.create_nodes()
+        self.factory = APIRequestFactory()
 
     def tearDown(self):
         self.r.flushdb()
@@ -70,6 +75,78 @@ def test_rsync_dir_with_nodes_when_rsync_fails(self, mock_os_system):
         with self.assertRaises(FileSynchronizationException):
             rsync_dir_with_nodes(settings.WORKLOAD_METRICS_DIR)
 
+    @mock.patch('api.common_utils.get_keystone_admin_auth')
+    def test_is_valid_request_new_valid_token(self, mock_keystone_admin_auth):
+        not_expired_admin_token = FakeTokenData((datetime.utcnow() + timedelta(minutes=5)).strftime('%Y-%m-%dT%H:%M:%SZ'),
+                                                {'roles': [{'name': 'admin'}, {'name': '_member_'}]})
+        mock_keystone_admin_auth.return_value.tokens.validate.return_value = not_expired_admin_token
+        request = self.factory.get('/')
+        request.META['HTTP_X_AUTH_TOKEN'] = 'new_not_expired_token'
+        resp = is_valid_request(request)
+        self.assertEquals(resp, 'new_not_expired_token')
+        self.assertTrue(mock_keystone_admin_auth.called)
+        mock_keystone_admin_auth.reset_mock()
+
+        # Successive calls should not invoke keystone
+        request = self.factory.get('/')
+        request.META['HTTP_X_AUTH_TOKEN'] = 'new_not_expired_token'
+        resp = is_valid_request(request)
+        self.assertEquals(resp, 'new_not_expired_token')
+        self.assertFalse(mock_keystone_admin_auth.called)
+
+    @mock.patch('api.common_utils.get_keystone_admin_auth')
+    def test_is_valid_request_new_expired_token(self, mock_keystone_admin_auth):
+        not_expired_admin_token = FakeTokenData((datetime.utcnow() - timedelta(minutes=5)).strftime('%Y-%m-%dT%H:%M:%SZ'),
+                                                {'roles': [{'name': 'admin'}, {'name': '_member_'}]})
+        mock_keystone_admin_auth.return_value.tokens.validate.return_value = not_expired_admin_token
+        request = self.factory.get('/')
+        request.META['HTTP_X_AUTH_TOKEN'] = 'expired_token'
+        resp = is_valid_request(request)
+        self.assertFalse(resp)
+
+    @mock.patch('api.common_utils.get_keystone_admin_auth')
+    def test_is_valid_request_not_admin(self, mock_keystone_admin_auth):
+        not_expired_admin_token = FakeTokenData((datetime.utcnow() + timedelta(minutes=5)).strftime('%Y-%m-%dT%H:%M:%SZ'),
+                                                {'roles': [{'name': '_member_'}]})
+        mock_keystone_admin_auth.return_value.tokens.validate.return_value = not_expired_admin_token
+        request = self.factory.get('/')
+        request.META['HTTP_X_AUTH_TOKEN'] = 'not_admin_token'
+        resp = is_valid_request(request)
+        self.assertFalse(resp)
+
+    @mock.patch('api.common_utils.get_keystone_admin_auth')
+    def test_is_valid_request_raises_exception(self, mock_keystone_admin_auth):
+        mock_keystone_admin_auth.return_value.tokens.validate.side_effect = Exception()
+        request = self.factory.get('/')
+        request.META['HTTP_X_AUTH_TOKEN'] = 'token'
+        resp = is_valid_request(request)
+        self.assertFalse(resp)
+
+    @mock.patch('api.common_utils.get_keystone_admin_auth')
+    def test_get_project_list_ok(self, mock_keystone_admin_auth):
+        fake_tenants_list = [FakeTenantData('1234567890abcdef', 'tenantA'), FakeTenantData('abcdef1234567890', 'tenantB')]
+        mock_keystone_admin_auth.return_value.tenants.list.return_value = fake_tenants_list
+        resp = get_project_list('token')
+        self.assertEquals(resp['1234567890abcdef'], 'tenantA')
+        self.assertEquals(resp['abcdef1234567890'], 'tenantB')
+
+    @override_settings(MANAGEMENT_ACCOUNT='mng_account', MANAGEMENT_ADMIN_USERNAME='mng_username', MANAGEMENT_ADMIN_PASSWORD='mng_pw',
+                       KEYSTONE_URL='http://localhost:35357/v2.0')
+    @mock.patch('api.common_utils.keystone_client.Client')
+    def test_get_keystone_admin_auth_ok(self, mock_keystone_client):
+        get_keystone_admin_auth()
+        mock_keystone_client.assert_called_with(username='mng_username', tenant_name='mng_account', password='mng_pw', auth_url='http://localhost:35357/v2.0')
+
+    def test_startup_run_ok(self):
+        self.create_startup_fixtures()
+        startup_run()
+        self.assertEquals(self.r.hget('workload_metric:1', 'enabled'), 'False')
+        self.assertEquals(self.r.hget('workload_metric:2', 'enabled'), 'False')
+        self.assertFalse(self.r.exists('metric:metric1'))
+        self.assertFalse(self.r.exists('metric:metric2'))
+        self.assertEquals(self.r.hget('policy:1', 'alive'), 'False')
+        self.assertEquals(self.r.hget('policy:2', 'alive'), 'False')
+
     #
     # URL tests
     #
@@ -111,4 +188,28 @@ def create_nodes(self):
     def configure_usernames_and_passwords_for_nodes(self):
         self.r.hmset('node:controller', {'ssh_username': 'user1', 'ssh_password': 's3cr3t'})
         self.r.hmset('node:storagenode1', {'ssh_username': 'user1', 'ssh_password': 's3cr3t'})
-        self.r.hmset('node:storagenode2', {'ssh_username': 'user1', 'ssh_password': 's3cr3t'})
+        self.r.hmset('node:storagenode2', {'ssh_username': 'user1', 'ssh_password': 's3cr3t'})
+
+    def create_startup_fixtures(self):
+        self.r.hmset('workload_metric:1', {'metric_name': 'm1.py', 'class_name': 'Metric1', 'execution_server': 'proxy', 'out_flow':'False',
+                                           'in_flow': 'False', 'enabled': 'True', 'id': '1'})
+        self.r.hmset('workload_metric:2', {'metric_name': 'm2.py', 'class_name': 'Metric2', 'execution_server': 'proxy', 'out_flow':'False',
+                                           'in_flow': 'False', 'enabled': 'True', 'id': '2'})
+        self.r.hmset('metric:metric1', {'network_location': '?', 'type': 'integer'})
+        self.r.hmset('metric:metric2', {'network_location': '?', 'type': 'integer'})
+        self.r.hmset('policy:1',
+                     {'alive': 'True', 'policy_description': 'FOR TENANT:0123456789abcdef DO SET compression'})
+        self.r.hmset('policy:2',
+                     {'alive': 'True', 'policy_description': 'FOR TENANT:0123456789abcdef DO SET encryption'})
+
+
+class FakeTokenData:
+    def __init__(self, expires, user):
+        self.expires = expires
+        self.user = user
+
+
+class FakeTenantData:
+    def __init__(self, id, name):
+        self.id = id
+        self.name = name

api/docs/_build/html/_sources/index.txt

@@ -1,9 +1,9 @@
-.. SDS Controller for Object Storage documentation master file, created by
+.. Crystal Controller documentation master file, created by
    sphinx-quickstart on Tue Dec 22 15:35:18 2015.
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to SDS Controller for Object Storage's documentation!
+Welcome to Crystal Controller documentation!
 =============================================================
 
 Contents:

api/docs/conf.py

@@ -24,12 +24,6 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #sys.path.insert(0, os.path.abspath('.'))
 sys.path.insert(0, os.path.abspath('..'))
-
-# dist_packages_path = os.path.join(os.sep, 'usr', 'lib', 'python2.7', 'dist-packages')
-# sys.path.insert(0, dist_packages_path)
-# local_dist_packages_path = os.path.join(os.sep, 'usr', 'local', 'lib', 'python2.7', 'dist-packages')
-# sys.path.insert(0, local_dist_packages_path)
-
 from django.conf import settings
 settings.configure()
 
@@ -61,17 +55,17 @@
 
 # General information about the project.
 project = u'Crystal Controller'
-copyright = u'2015-2016, Edgar Zamora-Gómez, Raúl Gracia'
-author = u'Edgar Zamora-Gómez, Raúl Gracia'
+copyright = u'2015-2016, Josep Sampé, Gerard París, Edgar Zamora-Gómez, Raúl Gracia'
+author = u'Josep Sampé, Gerard París, Edgar Zamora-Gómez, Raúl Gracia'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = u'1.0.0'
+version = u'1.0'
 # The full version, including alpha/beta/rc tags.
-release = u'1.0.0'
+release = u'1.0.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -214,7 +208,7 @@
 #html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'SDSControllerforObjectStoragedoc'
+htmlhelp_basename = 'CrystalControllerforObjectStoragedoc'
 
 # -- Options for LaTeX output ---------------------------------------------
 
@@ -236,8 +230,8 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  (master_doc, 'SDSControllerforObjectStorage.tex', u'SDS Controller for Object Storage Documentation',
-   u'Edgar Zamora-Gómez, Raúl Gracia', 'manual'),
+  (master_doc, 'CrystalControllerforObjectStorage.tex', u'Crystal Controller Documentation',
+   u'Josep Sampé, Gerard París, Edgar Zamora-Gómez, Raúl Gracia', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -266,7 +260,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'sdscontrollerforobjectstorage', u'SDS Controller for Object Storage Documentation',
+    (master_doc, 'crystalcontrollerforobjectstorage', u'Crystal Controller Documentation',
      [author], 1)
 ]
 
@@ -280,8 +274,8 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  (master_doc, 'SDSControllerforObjectStorage', u'SDS Controller for Object Storage Documentation',
-   author, 'SDSControllerforObjectStorage', 'One line description of project.',
+  (master_doc, 'CrystalControllerforObjectStorage', u'Crystal Controller Documentation',
+   author, 'CrystalControllerforObjectStorage', 'One line description of project.',
    'Miscellaneous'),
 ]
 
@@ -297,13 +291,14 @@
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
 
-# -- Mocks
+# -- Mocks: mock objects are added for those modules that are not declared in requirements.txt
+# Note: when using a submodule like pyactive.controller, both module and submodule must be declared.
 
 class Mock(MagicMock):
     @classmethod
     def __getattr__(cls, name):
             return Mock()
 
 if on_rtd:
-    MOCK_MODULES = ['pyactive.controller', 'pyactive.controller']
+    MOCK_MODULES = ['pyactive', 'pyactive.controller']
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)

api/filters/tests.py

@@ -8,7 +8,7 @@
 from rest_framework import status
 from rest_framework.test import APIRequestFactory
 
-from .views import dependency_list, dependency_detail, storlet_list, storlet_detail, storlet_list_deployed, filter_deploy, StorletData
+from .views import dependency_list, dependency_detail, storlet_list, storlet_detail, storlet_list_deployed, filter_deploy, unset_filter, StorletData
 
 
 # Tests use database=10 instead of 0.
@@ -126,15 +126,9 @@ def test_create_storlet_ok(self, mock_is_valid_request):
         response = storlet_list(request)
         storlets = json.loads(response.content)
         self.assertEqual(len(storlets), 2)
-
-        if storlets[0]['id'] == "1":
-            storlet1 = storlets[0]
-            storlet2 = storlets[1]
-        else:
-            storlet1 = storlets[1]
-            storlet2 = storlets[0]
-        self.assertEqual(storlet1['main'], 'com.example.FakeMain')
-        self.assertEqual(storlet2['main'], 'com.example.SecondMain')
+        sorted_list = sorted(storlets, key=lambda st: st['id'])
+        self.assertEqual(sorted_list[0]['main'], 'com.example.FakeMain')
+        self.assertEqual(sorted_list[1]['main'], 'com.example.SecondMain')
 
     def test_create_storlets_are_sorted_by_id(self, mock_is_valid_request):
         """
@@ -400,6 +394,18 @@ def test_delete_dependency_ok(self, mock_is_valid_request):
         dependencies = json.loads(response.content)
         self.assertEqual(len(dependencies), 0)
 
+    @mock.patch('filters.views.swift_client.delete_object')
+    def test_unset_filter_ok(self, mock_delete_object, mock_is_valid_request):
+        data20 = {'filter_name': 'XXXXX'}
+        data21 = {'filter_name': 'test-1.0.jar'}
+        self.r.hmset('pipeline:AUTH_0123456789abcdef', {'20': json.dumps(data20), '21': json.dumps(data21)})
+        unset_filter(self.r, '0123456789abcdef', {'filter_type':'storlet', 'filter_name': 'test-1.0.jar'}, 'fake_token')
+        mock_delete_object.assert_called_with(settings.SWIFT_URL + settings.SWIFT_API_VERSION + "/AUTH_0123456789abcdef",
+                                           'fake_token', "storlet", "test-1.0.jar", mock.ANY, mock.ANY, mock.ANY,
+                                           mock.ANY, mock.ANY)
+        self.assertFalse(self.r.hexists("pipeline:AUTH_0123456789abcdef", "21"))  # 21 was deleted
+        self.assertTrue(self.r.hexists("pipeline:AUTH_0123456789abcdef", "20"))  # 20 was not deleted
+
     #
     # Aux methods
     #

api/registry/dsl_parser.py

@@ -76,8 +76,8 @@ def parse(input_string):
     # TRANSIENT
     transient = Literal("TRANSIENT")
     action = Group(action("action") + oneOf(sfilter)("filter") +
-                   Optional(with_params + params_list("params") +
-                            Optional(Suppress("ON")+server_execution("server_execution"))) +
+                   Optional(with_params + params_list("params")) +
+                   Optional(Suppress("ON")+server_execution("server_execution")) +
                    Optional(transient("transient")))
 
     action_list = Group(delimitedList(action))

api/registry/dynamic_policies/consumer.py

@@ -19,11 +19,10 @@ def __init__(self, host, port, username, password, exchange, queue, routing_key,
         self.obj = obj
         self.queue = queue
 
-        print '- Exchange:', exchange
-        # result = channel.queue_declare(exclusive=True)
+        print '- Metric, Exchange:', exchange
+        print '- Metric, Routing_key: ', routing_key
+
         self._channel.queue_declare(queue=queue)
-        # queue_name = result.method.queue
-        print '- Routing_key: ', routing_key
 
         if routing_key:
             self._channel.queue_bind(exchange=exchange,
@@ -39,12 +38,12 @@ def callback(self, ch, method, properties, body):
         self.obj.notify(body)
 
     def start_consuming(self):
-        print '- Start to consume from rabbitmq'
+        print '- Metric, Start to consume from rabbitmq'
         self.thread = Thread(target=self._channel.start_consuming)
         self.thread.start()
 
     def stop_consuming(self):
-        print '- Stopping to consume from rabbitmq'
+        print '- Metric, Stopping to consume from rabbitmq'
         self._atom.stop()
         self._channel.stop_consuming()
         self._channel.close()

api/registry/dynamic_policies/metrics/abstract_metric.py

@@ -43,15 +43,15 @@ def attach(self, observer):
         :type observer: **any** PyActive Proxy type
         """
         # TODO: Add the possibility to subscribe to container or object
-        print ' - Metric attaching, observer: ', observer
+        print '- Metric, Attaching observer: ', observer
         tenant = observer.get_target()
 
         if tenant not in self._observers.keys():
             self._observers[tenant] = set()
         if observer not in self._observers[tenant]:
             self._observers[tenant].add(observer)
 
-    def detach(self, observer):
+    def detach(self, observer, target):
         """
         Asyncronous method. This method allows to be called remotelly.
         It is called from observers in order to unsubscribe from this workload
@@ -60,9 +60,9 @@ def detach(self, observer):
         :param observer: The PyActive proxy of the oberver rule that calls this method.
         :type observer: **any** PyActive Proxy type
         """
-        tenant = observer.get_target()
+        print '- Metric, Detaching observer: ', observer
         try:
-            self._observers[tenant].remove(observer)
+            self._observers[target].remove(observer)
         except KeyError:
             pass
 
@@ -77,7 +77,6 @@ def init_consum(self):
                            consumer appear.
         """
         try:
-            print '- Starting consumer'
             self.redis.hmset("metric:"+self.name, {"network_location": self._atom.aref.replace("atom:", "tcp:", 1), "type": "integer"})
 
             self.consumer = self.host.spawn_id(self.id + "_consumer",