Merge pull request #199 from ferbncode/dump-tests

CB-286: Add tests for dump manager functions

critiquebrainz/data/dump_manager.py

@@ -1,6 +1,5 @@
 from time import gmtime, strftime
 from datetime import datetime
-from functools import wraps
 import subprocess
 import tempfile
 import tarfile
@@ -12,9 +11,9 @@
 from flask.json import JSONEncoder
 import sqlalchemy
 import click
-from critiquebrainz.data.utils import create_path, remove_old_archives, slugify, explode_db_uri
+from critiquebrainz.data.utils import create_path, remove_old_archives, slugify, explode_db_uri, with_request_context
 from critiquebrainz.db import license as db_license, review as db_review
-from critiquebrainz import frontend, db
+from critiquebrainz import db
 
 
 cli = click.Group()
@@ -68,14 +67,6 @@
 }
 
 
-def with_request_context(f):
-    @wraps(f)
-    def decorated(*args, **kwargs):
-        with frontend.create_app().test_request_context():
-            return f(*args, **kwargs)
-    return decorated
-
-
 def has_data(table_name):
     with db.engine.connect() as connection:
         result = connection.execute(sqlalchemy.text("""

critiquebrainz/data/utils.py

@@ -5,7 +5,9 @@
 import sys
 import os
 import re
+from functools import wraps
 from critiquebrainz import db
+from critiquebrainz import frontend
 
 ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'admin', 'sql')
 
@@ -93,3 +95,23 @@ def remove_old_archives(location, pattern, is_dir=False, sort_key=None):
             shutil.rmtree(entry)
         else:
             os.remove(entry)
+
+
+def with_request_context(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        with frontend.create_app().test_request_context():
+            return f(*args, **kwargs)
+    return decorated
+
+
+def with_test_request_context(f):
+    """Decorator for providing request context for application using test_config.py."""
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        with frontend.create_app(
+            config_path=os.path.join(
+                os.path.dirname(os.path.realpath(__file__)),
+                '..', 'test_config.py')).test_request_context():
+            return f(*args, **kwargs)
+    return decorated

critiquebrainz/db/dump_manager_test.py

@@ -0,0 +1,89 @@
+import os
+import tempfile
+from datetime import datetime
+from click.testing import CliRunner
+from critiquebrainz.data.testing import DataTestCase
+from critiquebrainz.data import utils
+import critiquebrainz.db.license as db_license
+import critiquebrainz.db.users as db_users
+import critiquebrainz.db.review as db_review
+from critiquebrainz.db.user import User
+
+utils.with_request_context = utils.with_test_request_context  # noqa
+from critiquebrainz.data import dump_manager # pylint:disable=wrong-import-position
+
+
+def get_archives(root_dir):
+    """Returns a dictionary of bz2 archives and their respective paths.
+
+    Args:
+        root_dir: Path of the root dir.
+    Returns:
+        Dictionary with the following structure:
+        {
+            archive_name: archive_path
+        }
+    """
+    archives = {}
+    for roots, _, files in os.walk(root_dir):
+        for f in files:
+            if f.endswith('tar.bz2'):
+                archives[f] = os.path.join(roots, f)
+    return archives
+
+
+class DumpManagerTestCase(DataTestCase):
+
+    def setUp(self):
+        super().setUp()
+        self.tempdir = tempfile.mkdtemp()
+        self.runner = CliRunner()
+        self.license = db_license.create(
+            id=u'test',
+            full_name=u"Test License",
+        )
+
+    def test_full_db(self):
+        self.runner.invoke(dump_manager.full_db, ['--location', self.tempdir])
+        self.assertTrue(os.listdir(self.tempdir)[0].endswith('.tar.bz2'))
+
+    def test_json_dumps(self):
+        date = datetime.today().strftime('%Y%m%d')
+        self.runner.invoke(dump_manager.json, ['--location', self.tempdir])
+        self.assertIn(f'critiquebrainz-{date}-{self.license["id"]}-json.tar.bz2', os.listdir(self.tempdir))
+
+    def test_public(self):
+        self.runner.invoke(dump_manager.public, ['--location', self.tempdir])
+        archives = get_archives(self.tempdir).keys()
+        self.assertIn('cbdump.tar.bz2', archives)
+        self.assertIn('cbdump-reviews-all.tar.bz2', archives)
+        self.assertIn(f'cbdump-reviews-{self.license["id"]}.tar.bz2', archives)
+
+    def test_importer(self):
+        user = User(db_users.get_or_create("Tester", new_user_data={
+            "display_name": "test user",
+        }))
+        review = db_review.create(
+            user_id=user.id,
+            entity_id="e7aad618-fa86-3983-9e77-405e21796eca",
+            entity_type="release_group",
+            text="Testing",
+            rating=5,
+            is_draft=False,
+            license_id=self.license["id"],
+        )
+
+        # Make dumps and delete entities
+        self.runner.invoke(dump_manager.public, ['--location', self.tempdir])
+        archives = get_archives(self.tempdir)
+        db_review.delete(review['id'])
+        db_users.delete(user.id)
+        self.assertEqual(db_users.total_count(), 0)
+        self.assertEqual(db_review.get_count(), 0)
+
+        # Import dumps - cbdump.tar.bz2 and cbdump-reviews-all.tar.bz2 and check if data imported properly
+        self.runner.invoke(dump_manager.importer, [archives['cbdump.tar.bz2']])
+        self.assertEqual(db_users.total_count(), 1)
+
+        self.runner.invoke(dump_manager.importer, [archives['cbdump-reviews-all.tar.bz2']])
+        self.assertEqual(db_review.get_count(), 1)

docker/Dockerfile.test

@@ -20,6 +20,16 @@ RUN apt-get update \
                         libxml2-dev \
                         libxslt1-dev \
     && rm -rf /var/lib/apt/lists/*
+
+# PostgreSQL client
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+ENV PG_MAJOR 9.5
+RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
+RUN apt-get update \
+    && apt-get install -y postgresql-client-$PG_MAJOR \
+    && rm -rf /var/lib/apt/lists/*
+ENV PGPASSWORD "critiquebrainz"
+
 COPY requirements.txt /code/
 RUN pip install -r requirements.txt