Fix for running without optional packages

.github/workflows/codeql-analysis.yml

@@ -14,6 +14,10 @@ on:
   schedule:
     - cron: '0 2 * * 5'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze
@@ -47,7 +51,7 @@ jobs:
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file. 
+        # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
 

.github/workflows/docs.yml

@@ -2,6 +2,11 @@ name: Docs
 
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+
 jobs:
   docs:
     runs-on: ubuntu-latest

.github/workflows/lint.yml

@@ -2,6 +2,11 @@ name: Lint
 
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+
 jobs:
   flake8:
     name: flake8

.github/workflows/test.yml

@@ -2,6 +2,11 @@ name: Tests
 
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+
 jobs:
   unit-tests:
     runs-on: ${{ matrix.os }}
@@ -18,7 +23,44 @@ jobs:
             django-version: '3.2'
           - python-version: '3.11'
             django-version: '4.0'
+    env:
+        ENABLE_TASKS: no
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements/base.txt
+          pip install Django==${{ matrix.django-version }}
+          python setup.py install
+
+      - name: Run tests
+        run: coverage run --source=explorer manage.py test
 
+      - name: Upload Coverage to Codecov
+        uses: codecov/codecov-action@v3
+
+  unit-tests-with-optional-packages:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+        django-version: ['3.2', '4.0', '4.1']
+        os: [
+          ubuntu-20.04,
+        ]
+        exclude:
+          - python-version: '3.11'
+            django-version: '3.2'
+          - python-version: '3.11'
+            django-version: '4.0'
+    env:
+        ENABLE_TASKS: yes
     steps:
       - uses: actions/checkout@v1
       - name: Set up Python ${{ matrix.python-version }}

explorer/tests/test_exporters.py

@@ -1,4 +1,5 @@
 import json
+import unittest
 from datetime import date, datetime
 
 from django.core.serializers.json import DjangoJSONEncoder
@@ -10,6 +11,7 @@
 from explorer.exporters import CSVExporter, ExcelExporter, JSONExporter
 from explorer.models import QueryResult
 from explorer.tests.factories import SimpleQueryFactory
+from explorer.utils import is_xls_writer_available
 
 
 class TestCsv(TestCase):
@@ -76,6 +78,7 @@ def test_writing_datetimes(self):
 
 class TestExcel(TestCase):
 
+    @unittest.skipIf(not is_xls_writer_available(), 'excel exporter not available')
     def test_writing_excel(self):
         """
         This is a pretty crap test. It at least exercises the code.
@@ -109,6 +112,7 @@ def test_writing_excel(self):
 
         self.assertEqual(res[:2], expected)
 
+    @unittest.skipIf(not is_xls_writer_available(), 'excel exporter not available')
     def test_writing_dict_fields(self):
         res = QueryResult(
             SimpleQueryFactory(

explorer/tests/test_models.py

@@ -1,9 +1,11 @@
+import unittest
 from unittest.mock import Mock, patch
 
 from django.core.exceptions import ValidationError
 from django.db import connections
 from django.test import TestCase
 
+from explorer import app_settings
 from explorer.app_settings import EXPLORER_DEFAULT_CONNECTION as CONN
 from explorer.models import ColumnHeader, ColumnSummary, Query, QueryLog, QueryResult
 from explorer.tests.factories import SimpleQueryFactory
@@ -78,6 +80,7 @@ def test_log_saves_duration(self):
         log = QueryLog.objects.first()
         self.assertEqual(log.duration, res.duration)
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.models.s3_url')
     @patch('explorer.models.get_s3_bucket')
     def test_get_snapshots_sorts_snaps(self, mocked_get_s3_bucket, mocked_s3_url):

explorer/tests/test_schema.py

@@ -1,10 +1,11 @@
+import unittest
 from unittest.mock import patch
 
 from django.core.cache import cache
 from django.db import connection
 from django.test import TestCase
 
-from explorer import schema
+from explorer import app_settings, schema
 from explorer.app_settings import EXPLORER_DEFAULT_CONNECTION as CONN
 
 
@@ -70,6 +71,7 @@ def test_app_exclude_views(self, mocked_include_views):
         tables = [x[0] for x in res]
         self.assertNotIn(database_view, tables)
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.schema.do_async')
     def test_builds_async(self, mocked_async_check):
         mocked_async_check.return_value = True

explorer/tests/test_tasks.py

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
+import unittest
 from datetime import datetime, timedelta
 from io import StringIO
 from unittest.mock import patch
 
 from django.core import mail
 from django.test import TestCase
 
+from explorer import app_settings
 from explorer.app_settings import EXPLORER_DEFAULT_CONNECTION as CONN
 from explorer.models import QueryLog
 from explorer.tasks import build_schema_cache_async, execute_query, snapshot_queries, truncate_querylogs
@@ -14,6 +16,7 @@
 
 class TestTasks(TestCase):
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.tasks.s3_upload')
     def test_async_results(self, mocked_upload):
         mocked_upload.return_value = 'http://s3.com/your-file.csv'
@@ -39,6 +42,7 @@ def test_async_results(self, mocked_upload):
         )
         self.assertEqual(mocked_upload.call_count, 1)
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.tasks.s3_upload')
     def test_async_results_fails_with_message(self, mocked_upload):
         mocked_upload.return_value = 'http://s3.com/your-file.csv'
@@ -53,6 +57,7 @@ def test_async_results_fails_with_message(self, mocked_upload):
         self.assertIn('[SQL Explorer] Error ', mail.outbox[1].subject)
         self.assertEqual(mocked_upload.call_count, 0)
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.tasks.s3_upload')
     def test_snapshots(self, mocked_upload):
         mocked_upload.return_value = 'http://s3.com/your-file.csv'
@@ -65,6 +70,7 @@ def test_snapshots(self, mocked_upload):
         snapshot_queries()
         self.assertEqual(mocked_upload.call_count, 3)
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     def test_truncating_querylogs(self):
         QueryLog(sql='foo').save()
         QueryLog.objects.filter(sql='foo').update(
@@ -77,6 +83,7 @@ def test_truncating_querylogs(self):
         truncate_querylogs(30)
         self.assertEqual(QueryLog.objects.count(), 1)
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.schema.build_schema_info')
     def test_build_schema_cache_async(self, mocked_build):
         mocked_build.return_value = ['list_of_tuples']

explorer/tests/test_views.py

@@ -1,6 +1,7 @@
 import importlib
 import json
 import time
+import unittest
 from unittest.mock import Mock, patch
 
 from django.contrib.auth.models import User
@@ -280,6 +281,7 @@ def test_user_query_views(self):
         with self.settings(EXPLORER_USER_QUERY_VIEWS={99: [111, 123]}):
             self.assertTrue(user_can_see_query(request, **kwargs))
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.models.get_s3_bucket')
     def test_query_snapshot_renders(self, mocked_conn):
         conn = Mock()
@@ -654,6 +656,7 @@ def test_admin_required(self):
         )
         self.assertTemplateUsed(resp, 'admin/login.html')
 
+    @unittest.skipIf(not app_settings.ENABLE_TASKS, 'tasks not enabled')
     @patch('explorer.schema.do_async')
     def test_builds_async(self, mocked_async_check):
         mocked_async_check.return_value = True

explorer/utils.py

@@ -6,7 +6,6 @@
 from django.contrib.auth.forms import AuthenticationForm
 from django.contrib.auth.views import LoginView
 
-import boto3
 import sqlparse
 from sqlparse import format as sql_format
 from sqlparse.sql import Token, TokenList
@@ -187,6 +186,7 @@ def get_valid_connection(alias=None):
 
 
 def get_s3_bucket():
+    import boto3
     kwargs = {
         'aws_access_key_id': app_settings.S3_ACCESS_KEY,
         'aws_secret_access_key': app_settings.S3_SECRET_KEY,
@@ -210,3 +210,11 @@ def s3_url(bucket, key):
         Params={'Bucket': app_settings.S3_BUCKET, 'Key': key},
         ExpiresIn=app_settings.S3_LINK_EXPIRATION)
     return url
+
+
+def is_xls_writer_available():
+    try:
+        import xlsxwriter  # noqa
+        return True
+    except ImportError:
+        return False

requirements/base.txt

@@ -1,2 +1,3 @@
 sqlparse>=0.4.0
 coverage
+factory-boy>=3.1.0

requirements/optional.txt

@@ -2,6 +2,5 @@ importlib-metadata<5.0; python_version <= '3.7'
 celery>=4.0
 boto3>=1.20.0
 xlsxwriter>=1.3.6
-factory-boy>=3.1.0
 matplotlib<4
 seaborn<0.12

test_project/__init__.py

@@ -1,3 +1,6 @@
-from .celery_config import app as celery_app
+try:
+    from .celery_config import app as celery_app
 
-__all__ = ['celery_app']
+    __all__ = ['celery_app']
+except ImportError:
+    pass

test_project/settings.py

@@ -95,5 +95,5 @@
 )
 
 EXPLORER_USER_QUERY_VIEWS = {}
-EXPLORER_TASKS_ENABLED = True
+EXPLORER_TASKS_ENABLED = os.environ.get('ENABLE_TASKS') == 'yes'  # variable set in github action - assumed to be str
 EXPLORER_S3_BUCKET = 'thisismybucket.therearemanylikeit.butthisoneismine'