Merge 5b3a856 into 766a5dd

.travis.yml

@@ -4,6 +4,10 @@ cache: pip
 dist: xenial
 services:
   - docker
+addons:
+apt:
+  packages:
+    - python-mysqldb
 install:
   - pip install --upgrade pip
   - pip install --upgrade setuptools pipenv

Pipfile

@@ -6,10 +6,16 @@ verify_ssl = true
 [dev-packages]
 bandit = "==1.4.0"
 coveralls = "==1.2.0"
+# The Cython dependency of pymssql is not correctly resolved on all systems
+# so it is explicitly included here.
+cython = "==0.29.2"
 flake8 = "==3.5.0"
 isort = "==4.3.4"
+mysqlclient = "==1.3.14"
+psycopg2 = "==2.7.5"
 pydocstyle = "==2.1.1"
 pylint = "==1.8.2"
+pymssql = "==2.1.4"
 pyorient = "==1.5.5"
 pytest = "==4.0.2"
 pytest-cov = "==2.5.1"

docker-compose.yml

@@ -8,3 +8,33 @@ services:
       - "127.0.0.1:2424:2424"
     environment:
       ORIENTDB_ROOT_PASSWORD: root
+  postgres:
+    image: postgres:10.5
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: root
+    ports:
+      - "127.0.0.1:5432:5432"
+  mysql:
+    image: mysql:8.0.11
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports:
+      - "127.0.0.1:3306:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+  mariadb:
+    image: mariadb:10.3.11
+    restart: always
+    ports:
+      - "127.0.0.1:3307:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+  mssql:
+    image: mcr.microsoft.com/mssql/server:2017-latest
+    restart: always
+    ports:
+      - "127.0.0.1:1433:1433"
+    environment:
+      ACCEPT_EULA: "yes"
+      MSSQL_SA_PASSWORD: Root-secure1  # password requirements are more stringent for MSSQL image

graphql_compiler/tests/conftest.py

@@ -3,8 +3,14 @@
 import time
 
 import pytest
+import six
 
+from .test_data_tools.data_tool import (
+    generate_orient_integration_data, generate_orient_snapshot_data, generate_sql_integration_data,
+    init_sql_integration_test_backends, tear_down_integration_test_backends
+)
 from .test_data_tools.graph import get_test_graph
+from .test_data_tools.schema import load_schema
 
 
 # Pytest fixtures depend on name redefinitions to work,
@@ -13,15 +19,25 @@
 
 
 @pytest.fixture(scope='session')
-def init_graph():
+def init_snapshot_graph_client():
     """Return a client for an initialized db, with all test data imported."""
+    return _init_graph_client(load_schema, generate_orient_snapshot_data)
+
+
+@pytest.fixture(scope='session')
+def init_integration_graph_client():
+    """Return a client for an initialized db, with all test data imported."""
+    return _init_graph_client(load_schema, generate_orient_integration_data)
+
+
+def _init_graph_client(load_schema_func, generate_data_func):
     graph_name = 'animals'
 
     # Try to set up the database for the test up to 20 times before giving up.
     set_up_successfully = False
     for _ in range(20):
         try:
-            graph_client = get_test_graph(graph_name)
+            graph_client = get_test_graph(graph_name, load_schema_func, generate_data_func)
             set_up_successfully = True
             break
         except Exception as e:  # pylint: disable=broad-except
@@ -35,6 +51,41 @@ def init_graph():
 
 
 @pytest.fixture(scope='class')
-def graph_client(request, init_graph):
+def graph_client(request, init_snapshot_graph_client):
     """Get a client for an initialized db, with all test data imported."""
-    request.cls.graph_client = init_graph
+    request.cls.graph_client = init_snapshot_graph_client
+
+
+@pytest.fixture(scope='class')
+def integration_graph_client(request, init_integration_graph_client):
+    """Get a client for an initialized db, with all test data imported."""
+    request.cls.graph_client = init_integration_graph_client
+
+
+@pytest.fixture(scope='class')
+def sql_integration_data(request):
+    """Generate integration data for SQL backends."""
+    # initialize each SQL backend, and open a transaction on each one.
+    sql_test_backends = init_sql_integration_test_backends()
+    # write fixture test data within the transaction
+    generate_sql_integration_data(sql_test_backends)
+    # make sql backends accessible within the test class
+    request.cls.sql_test_backends = sql_test_backends
+    # yield the fixture to allow testing class to run
+    yield
+    # tear down the fixture after the testing class runs all tests
+    # including rolling back transaction to ensure all fixture data removed.
+    tear_down_integration_test_backends(sql_test_backends)
+
+
+@pytest.fixture(scope='function')
+def sql_integration_test(request):
+    """Open nested transaction before every test function, and rollback transaction afterwards."""
+    sql_test_backends = request.cls.sql_test_backends
+    test_transactions = []
+    for sql_test_backend in six.itervalues(sql_test_backends):
+        transaction = sql_test_backend.connection.begin_nested()
+        test_transactions.append(transaction)
+    yield
+    for transaction in test_transactions:
+        transaction.rollback()

graphql_compiler/tests/integration_tests/__init__.py

@@ -0,0 +1 @@
+# Copyright 2018-present Kensho Technologies, LLC.

graphql_compiler/tests/integration_tests/integration_backend_config.py

@@ -0,0 +1,40 @@
+# Copyright 2018-present Kensho Technologies, LLC.
+from collections import namedtuple
+
+from .. import test_backend
+
+
+DEFAULT_ROOT_PASSWORD = u'root'
+MSSQL_ROOT_PASSWORD = u'Root-secure1'  # mssql has stricter root password restrictions
+
+SQL_BACKENDS = {
+    test_backend.POSTGRES,
+    test_backend.MYSQL,
+    test_backend.MARIADB,
+    test_backend.MSSQL,
+    test_backend.SQLITE,
+}
+
+MATCH_BACKENDS = {
+    test_backend.ORIENTDB,
+}
+
+SQL_BACKEND_TO_CONNECTION_STRING = {
+    test_backend.POSTGRES:
+        u'postgresql://postgres:{password}@localhost:5432'.format(password=DEFAULT_ROOT_PASSWORD),
+    test_backend.MYSQL:
+        u'mysql://root:{password}@127.0.0.1:3306'.format(password=DEFAULT_ROOT_PASSWORD),
+    test_backend.MARIADB:
+        u'mysql://root:{password}@127.0.0.1:3307'.format(password=DEFAULT_ROOT_PASSWORD),
+    test_backend.MSSQL:
+        u'mssql+pymssql://SA:{password}@localhost:1433'.format(password=MSSQL_ROOT_PASSWORD),
+    test_backend.SQLITE:
+        u'sqlite:///:memory:',
+}
+
+SqlTestBackend = namedtuple('SqlTestBackend', (
+    'connection_string',
+    'engine',
+    'connection',
+    'transaction',
+))

graphql_compiler/tests/integration_tests/integration_test_helpers.py

@@ -0,0 +1,56 @@
+# Copyright 2018-present Kensho Technologies, LLC.
+from graphql import GraphQLString
+import six
+from sqlalchemy import text
+
+from ... import CompilationResult, OutputMetadata, graphql_to_match
+from ...compiler import SQL_LANGUAGE
+
+
+def sort_db_results(results):
+    """Deterministically sort DB results.
+
+    Args:
+        results: List[Dict], results from a DB.
+
+    Returns:
+        List[Dict], sorted DB results.
+    """
+    sort_order = []
+    if len(results) > 0:
+        sort_order = sorted(six.iterkeys(results[0]))
+
+    def sort_key(result):
+        """Convert None/Not None to avoid comparisons of None to a non-None type."""
+        return tuple((result[col] is not None, result[col]) for col in sort_order)
+
+    return sorted(results, key=sort_key)
+
+
+def compile_and_run_match_query(schema, graphql_query, parameters, graph_client):
+    """Compiles and runs a MATCH query against the supplied graph client."""
+    compilation_result = graphql_to_match(schema, graphql_query, parameters)
+    query = compilation_result.query
+    results = [row.oRecordData for row in graph_client.command(query)]
+    return results
+
+
+def compile_and_run_sql_query(schema, graphql_query, parameters, sql_test_backend):
+    """Compiles and runs a SQL query against the supplied SQL backend."""
+    # TODO: un-mock the SQL compilation once the SQL backend can run queries.
+    def mock_sql_compilation(schema, graphql_query, parameters, compiler_metadata):
+        """Mock out SQL backend compilation for unimplemented SQL backend."""
+        mock_compilation_result = CompilationResult(
+            query=text('SELECT name AS animal_name FROM animal'),
+            language=SQL_LANGUAGE,
+            input_metadata={},
+            output_metadata={'animal_name': OutputMetadata(GraphQLString, False)}
+        )
+        return mock_compilation_result
+
+    compilation_result = mock_sql_compilation(schema, graphql_query, parameters, None)
+    query = compilation_result.query
+    results = []
+    for result in sql_test_backend.connection.execute(query):
+        results.append(dict(result))
+    return results