improve test coverage (#1372)

.github/workflows/django-import-export-ci.yml

@@ -7,112 +7,80 @@ on:
   pull_request:
     branches:
     - main
+    # this is a temporary addition - can be removed after 3.0 release
+    - release-3-x
 
 jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      USERNAME: testuser
-      PASSWD: ${{ secrets.TEST_PASSWD }}
+      DB_NAME: import_export
+      IMPORT_EXPORT_POSTGRESQL_USER: postgres
+      IMPORT_EXPORT_POSTGRESQL_PASSWORD: somepass
+      IMPORT_EXPORT_MYSQL_USER: root
+      IMPORT_EXPORT_MYSQL_PASSWORD: root
     strategy:
-      max-parallel: 4
       matrix:
-        db: [ sqlite, postgres, mysql ]
-        python-version: [ 3.7, 3.8, 3.9, "3.10" ]
-        django-version: [ 3.2, 4.0, main ]
-        include:
-          - db: postgres
-            db_port: 5432
-          - db: mysql
-            db_port: 3306
-        exclude:
-          - django-version: main
-            python-version: 3.7
-          - django-version: 4.0
-            python-version: 3.7
-          - django-version: 3.2
-            python-version: "3.10"
+        python-version:
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
     services:
-      mysql:
-        image: mysql:8.0
-        env:
-          IMPORT_EXPORT_TEST_TYPE: mysql-innodb
-          IMPORT_EXPORT_MYSQL_USER: ${{ env.TESTUSER }}
-          IMPORT_EXPORT_MYSQL_PASSWORD: ${{ env.PASSWD }}
-          MYSQL_USER: ${{ env.TESTUSER }}
-          MYSQL_PASSWORD: ${{ env.IMPORT_EXPORT_MYSQL_PASSWORD }}
-          MYSQL_ROOT_PASSWORD: root
-          MYSQL_DATABASE: import_export
-        ports:
-            - 3306:3306
-        options: >-
-          --health-cmd="mysqladmin ping"
-          --health-interval=10s
-          --health-timeout=5s
-          --health-retries=3
       postgres:
         image: postgres
         env:
-          IMPORT_EXPORT_TEST_TYPE: postgres
-          IMPORT_EXPORT_POSTGRESQL_USER: postgres
-          IMPORT_EXPORT_POSTGRESQL_PASSWORD: ${{ env.PASSWD }}
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: import_export
+          POSTGRES_USER: ${{ env.IMPORT_EXPORT_POSTGRESQL_USER }}
+          POSTGRES_PASSWORD: ${{ env.IMPORT_EXPORT_POSTGRESQL_PASSWORD }}
+          POSTGRES_DB: ${{ env.DB_NAME }}
         ports:
           - 5432:5432
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
     steps:
+    - name: Set up MySQL
+      run: >
+        sudo /etc/init.d/mysql start
+        
+        mysql -e 'CREATE DATABASE ${{ env.DB_NAME }};' 
+        -u${{ env.IMPORT_EXPORT_MYSQL_USER }}
+        -p${{ env.IMPORT_EXPORT_MYSQL_PASSWORD }}
     - name: Check out repository code
       uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Run isort checks
-      uses: jamescurtin/isort-action@master
+    - uses: actions/cache@v2
       with:
-        sortPaths: "import_export tests"
-        configuration: "--check-only"
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
     - name: Install Dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements/base.txt
-        pip install -r requirements/test.txt
-    - if: matrix.django-version != 'main'
-      name: Upgrade Django version (release)
-      run: |
-        python -m pip install "Django~=${{ matrix.django-version }}.0"
-    - if: matrix.django-version == 'main'
-      name: Upgrade Django version (main)
-      run: |
-        python -m pip install "https://github.com/django/django/archive/main.tar.gz"
-    - name: List versions
-      run: |
-        echo "Python ${{ matrix.python-version }} -> Django ${{ matrix.django-version }}"
-        python --version
-        echo "Django `django-admin --version`"
-    - name: Run Tests
+        python -m pip install --upgrade pip 
+        pip install tox tox-py coverage coveralls
+    - name: Run tox targets for ${{ matrix.python-version }} (sqlite)
+      run: tox --py current
+    - name: Run tox targets for ${{ matrix.python-version }} (postgres)
+      run: tox --py current
+      env:
+        IMPORT_EXPORT_TEST_TYPE: postgres
+    - name: Run tox targets for ${{ matrix.python-version }} (mysql)
+      run: tox --py current
       env:
-        DB: ${{ matrix.db }}
-        DB_HOST: 127.0.0.1
-        DB_PORT: ${{ matrix.db_port }}
-        DB_PASSWORD: ${{ env.PASSWD }}
-      run: >-
-        PYTHONPATH=".:tests:$PYTHONPATH" python
-        -W error::DeprecationWarning -W error::PendingDeprecationWarning
-        -m coverage run --omit='setup.py,./tests/*,./import_export/locale/*'
-        --source=. tests/manage.py test core --settings=
+        IMPORT_EXPORT_TEST_TYPE: mysql-innodb
+    - name: Combine test coverage
+      run: coverage combine
     - name: Upload coverage data to coveralls.io
       run: coveralls --service=github
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        COVERALLS_FLAG_NAME: ${{ matrix.db }}-${{ matrix.django-version }}-${{ matrix.python-version }}
+        COVERALLS_FLAG_NAME: ${{ matrix.python-version }}
         COVERALLS_PARALLEL: true
 
   coveralls:

docs/changelog.rst

@@ -10,20 +10,17 @@ Breaking changes
 This release makes the following changes to the API.  You may need to update your implementation to
 accommodate these changes.
 
+- Add support for multiple resources in ModelAdmin. (#1223)
+  - The `*Mixin.resource_class` accepting single resource has been deprecated (will work for few next versions) and the new `*Mixin.resource_classes` accepting subscriptable type (list, tuple, ...) has been added.
+  - Same applies to all of the `get_resource_class`, `get_import_resource_class` and `get_export_resource_class` methods.
+
 - Check value of ManyToManyField in skip_row() (#1271)
-   - This fixes an issue where ManyToMany fields are not checked correctly in `skip_row()`.
-     This means that `skip_row()` now takes `row` as a mandatory arg.
-     If you have overridden `skip_row()` in your own implementation, you will need to add `row`
-     as an arg.
+   - This fixes an issue where ManyToMany fields are not checked correctly in `skip_row()`.  This means that `skip_row()` now takes `row` as a mandatory arg.  If you have overridden `skip_row()` in your own implementation, you will need to add `row` as an arg.
 
 - Use 'create' flag instead of instance.pk (#1362)
-   - ``import_export.resources.save_instance()`` now takes an additional mandatory argument: `is_create`.
-     If you have over-ridden `save_instance()` in your own code, you will need to add this new argument.
+   - ``import_export.resources.save_instance()`` now takes an additional mandatory argument: `is_create`.  If you have over-ridden `save_instance()` in your own code, you will need to add this new argument.
 
-- Add support for multiple resources in ModelAdmin. (#1223)
-  - The `*Mixin.resource_class` accepting single resource has been deprecated (will work for few next versions) and
-    the new `*Mixin.resource_classes` accepting subscriptable type (list, tuple, ...) has been added.
-  - Same applies to all of the `get_resource_class`, `get_import_resource_class` and `get_export_resource_class` methods.
+- `exceptions.py` module is unused and has been deprecated.  This module will be removed in a future release (#1372)
 
 Enhancements
 ############
@@ -34,6 +31,7 @@ Development
 ###########
 
 - Drop support for python3.6, django 2.2, 3.0, 3.1 (#1366)
+- Increased test coverage, refactored CI build to use tox (#1372)
 
 2.7.1 (2021-12-23)
 ------------------

docs/conf.py

@@ -36,7 +36,7 @@
 
 # General information about the project.
 project = 'django-import-export'
-copyright = '2012–2020, Bojan Mihelac'
+copyright = '2012–2022, Bojan Mihelac'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/index.rst

@@ -32,6 +32,7 @@ exporting data with included admin integration.
    import_workflow
    bulk_import
    celery
+   testing
    changelog
 
 .. toctree::

docs/testing.rst

@@ -0,0 +1,58 @@
+Testing
+=======
+
+All tests can be run using `tox <https://tox.wiki/en/latest/>`_ simply by running the `tox` command.
+
+By default, tests are run against a local sqlite2 instance, however you can also use docker to test against MySQL and Postgres databases as follows.
+
+First, set environment variables.  Note that the ``IMPORT_EXPORT_TEST_TYPE`` must be set according to the type of tests you wish to run.  Set to 'postgres' for postgres tests, and 'mysql-innodb' for mysql tests.  If this environment variable is blank (or is any other value) then the default sqlite2 db will be used.
+
+.. code-block:: bash
+
+  export DJANGO_SETTINGS_MODULE=settings
+
+  export IMPORT_EXPORT_POSTGRESQL_USER=pguser
+  export IMPORT_EXPORT_POSTGRESQL_PASSWORD=pguserpass
+
+  export IMPORT_EXPORT_MYSQL_USER=mysqluser
+  export IMPORT_EXPORT_MYSQL_PASSWORD=mysqluserpass
+
+  # for testing against mysql db
+  export IMPORT_EXPORT_TEST_TYPE=mysql-innodb
+
+  # for testing against postgres db
+  export IMPORT_EXPORT_TEST_TYPE=postgres
+
+Assuming that you have docker installed on your system:
+
+.. code-block:: bash
+
+  docker-compose -f tests/docker-compose.yml up
+
+Now you can run all tests with: ``tox``
+
+To shutdown the database and delete all data:
+
+.. code-block:: bash
+
+  docker-compose -f tests/docker-compose.yml down -v
+
+Coverage
+########
+
+Coverage data is written in parallel mode by default (defined in ``setup.cfg``).  After a tox run, you can view coverage data as follows:
+
+.. code-block:: bash
+
+  # combine all coverage data generated by tox into one file
+  coverage combine
+
+  # produce an HTML coverage report
+  coverage html
+
+Check the output of the above commands to locate the coverage HTML file.
+
+Continuous Integration
+######################
+
+The steps defined above are automated for each build.  Refer to ``.github/workflows``)

import_export/exceptions.py

@@ -1,3 +1,8 @@
+import warnings
+
+warnings.warn("the exceptions module is deprecated and will be removed in a future release", DeprecationWarning)
+
+
 class ImportExportError(Exception):
     """A generic exception for all others to extend."""
     pass

import_export/formats/base_formats.py

@@ -1,6 +1,5 @@
-from importlib import import_module
-
 import tablib
+from tablib.formats import registry
 
 
 class Format:
@@ -61,14 +60,10 @@ def get_format(self):
         """
         Import and returns tablib module.
         """
-        try:
-            # Available since tablib 1.0
-            from tablib.formats import registry
-        except ImportError:
-            return import_module(self.TABLIB_MODULE)
-        else:
-            key = self.TABLIB_MODULE.split('.')[-1].replace('_', '')
-            return registry.get_format(key)
+        if not self.TABLIB_MODULE:
+            raise AttributeError("TABLIB_MODULE must be defined")
+        key = self.TABLIB_MODULE.split('.')[-1].replace('_', '')
+        return registry.get_format(key)
 
     @classmethod
     def is_available(cls):

import_export/resources.py

@@ -1078,7 +1078,7 @@ def widget_from_django_field(cls, f, default=widgets.Widget):
         Returns the widget that would likely be associated with each
         Django type.
 
-        Includes mapping of Postgres Array and JSON fields. In the case that
+        Includes mapping of Postgres Array field. In the case that
         psycopg2 is not installed, we consume the error and process the field
         regardless.
         """

requirements/test.txt

@@ -1,8 +1,8 @@
 isort
 psycopg2-binary
 mysqlclient
-coveralls
 chardet
 pytz
 memory-profiler
-django-extensions
+django-extensions
+coverage

setup.cfg

@@ -7,3 +7,7 @@ python-file-with-version = import_export/__init__.py
 
 [isort]
 profile = black
+
+[coverage:run]
+parallel = true
+source = import_export