Initial Druid mock unit tests

caravel/models.py

@@ -21,7 +21,7 @@
 from flask import flash, request, g
 from flask.ext.appbuilder import Model
 from flask.ext.appbuilder.models.mixins import AuditMixin
-from pydruid import client
+from pydruid.client import PyDruid
 from pydruid.utils.filters import Dimension, Filter
 from six import string_types
 from sqlalchemy import (
@@ -830,19 +830,21 @@ def __repr__(self):
         return self.cluster_name
 
     def get_pydruid_client(self):
-        cli = client.PyDruid(
+        cli = PyDruid(
             "http://{0}:{1}/".format(self.broker_host, self.broker_port),
             self.broker_endpoint)
         return cli
 
-    def refresh_datasources(self):
+    def get_datasources(self):
         endpoint = (
             "http://{obj.coordinator_host}:{obj.coordinator_port}/"
             "{obj.coordinator_endpoint}/datasources"
         ).format(obj=self)
 
-        datasources = json.loads(requests.get(endpoint).text)
-        for datasource in datasources:
+        return json.loads(requests.get(endpoint).text)
+
+    def refresh_datasources(self):
+        for datasource in self.get_datasources():
             DruidDatasource.sync_to_db(datasource, self)
 
 
@@ -950,9 +952,9 @@ def sync_to_db(cls, name, cluster):
         if not datasource:
             datasource = cls(datasource_name=name)
             session.add(datasource)
-            flash("Adding new datasource [{}]".format(name), "success")
+            logging.info("Adding new datasource [{}]".format(name))
         else:
-            flash("Refreshing datasource [{}]".format(name), "info")
+            logging.info("Refreshing datasource [{}]".format(name))
         datasource.cluster = cluster
 
         cols = datasource.latest_metadata()

caravel/views.py

@@ -365,13 +365,13 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin):  # noqa
         'created_by_', 'created_on',
         'changed_by_', 'changed_on',
         'offset']
-    order_columns = utils.list_minus(
-        list_columns, ['created_by_', 'changed_by_'])
-    related_views = [DruidColumnInlineView, DruidMetricInlineView]
+    related_views = [
+        DruidColumnInlineView, DruidMetricInlineView]
     edit_columns = [
         'datasource_name', 'cluster', 'description', 'owner',
         'is_featured', 'is_hidden', 'default_endpoint', 'offset',
         'cache_timeout']
+    add_columns = edit_columns
     page_size = 500
     base_order = ('datasource_name', 'asc')
     description_columns = {

dev-reqs.txt

@@ -1,4 +1,5 @@
 coveralls
+mock
 nose
 sphinx
 sphinx_bootstrap_theme

tests/core_tests.py

@@ -1,14 +1,22 @@
+"""Unit tests for Caravel"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from datetime import datetime
 import doctest
 import imp
 import os
 import unittest
+from mock import Mock, patch
 
 from flask import escape
 
 import caravel
 from caravel import app, db, models, utils, appbuilder
 
-os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config'
+#os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config'
 
 app.config['TESTING'] = True
 app.config['CSRF_ENABLED'] = False
@@ -18,10 +26,10 @@
 cli = imp.load_source('cli', BASE_DIR + "/bin/caravel")
 
 
-class CaravelTests(unittest.TestCase):
+class CoreTests(unittest.TestCase):
 
     def __init__(self, *args, **kwargs):
-        super(CaravelTests, self).__init__(*args, **kwargs)
+        super(CoreTests, self).__init__(*args, **kwargs)
         self.client = app.test_client()
         role_admin = appbuilder.sm.find_role('Admin')
         user = appbuilder.sm.find_user('admin')
@@ -89,5 +97,68 @@ def test_misc(self):
         assert self.client.get('/ping').data.decode('utf-8') == "OK"
 
 
+SEGMENT_METADATA = [{
+  "id" : "some_id",
+  "intervals" : [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ],
+  "columns" : {
+    "__time" : {
+        "type" : "LONG", "hasMultipleValues" : False,
+        "size" : 407240380, "cardinality" : None, "errorMessage" : None },
+    "dim1" : {
+        "type" : "STRING", "hasMultipleValues" : False,
+        "size" : 100000, "cardinality" : 1944, "errorMessage" : None },
+    "dim2" : {
+        "type" : "STRING", "hasMultipleValues" : True,
+        "size" : 100000, "cardinality" : 1504, "errorMessage" : None },
+    "metric1" : {
+        "type" : "FLOAT", "hasMultipleValues" : False,
+        "size" : 100000, "cardinality" : None, "errorMessage" : None }
+  },
+  "aggregators" : {
+    "metric1" : {
+        "type" : "longSum",
+        "name" : "metric1",
+        "fieldName" : "metric1" }
+  },
+  "size" : 300000,
+  "numRows" : 5000000
+}]
+
+
+class DruidTests(unittest.TestCase):
+
+    """Testing interactions with Druid"""
+
+    @patch('caravel.models.PyDruid')
+    def test_client(self, PyDruid):
+        from caravel.models import DruidCluster
+        instance = PyDruid.return_value
+        instance.time_boundary.return_value = [
+            {'result': {'maxTime': '2016-01-01'}}]
+        instance.segment_metadata.return_value = SEGMENT_METADATA
+
+        cluster = (
+            db.session
+            .query(DruidCluster)
+            .filter_by(cluster_name='test_cluster')
+            .first()
+        )
+        if cluster:
+            db.session.delete(cluster)
+        db.session.commit()
+
+        cluster = DruidCluster(
+            cluster_name='test_cluster',
+            coordinator_host='localhost',
+            coordinator_port=7979,
+            broker_host='localhost',
+            broker_port=7980,
+            metadata_last_refreshed=datetime.now())
+        db.session.add(cluster)
+        cluster.get_datasources = Mock(return_value=['test_datasource'])
+        cluster.refresh_datasources()
+        db.session.commit()
+
+
 if __name__ == '__main__':
     unittest.main()