Add support for PostgreSQL (second attempt)

.travis.yml

@@ -4,20 +4,32 @@ language: python
 
 cache: pip
 
+services: postgresql
+
 python:
   - 3.5
 
 env:
-  - AMY_ENABLE_PYDATA=true
-    AMY_PYDATA_USERNAME=username
-    AMY_PYDATA_PASSWORD=password
-  - CHECK_MIGRATION=true
+  # DB and PYDATA envvars are not used anywhere, they're only to make it easier
+  # to distinguish different builds.
+  - PYDATA=false DB=sqlite3
+    CHECK_MIGRATION=true
+  - PYDATA=false DB=postgres
+    CHECK_MIGRATION=true
+    DATABASE_URL="postgres://postgres:@localhost/testdb"
+  - PYDATA=true DB=sqlite3
+    AMY_ENABLE_PYDATA=true AMY_PYDATA_USERNAME=username AMY_PYDATA_PASSWORD=password
+  - PYDATA=true DB=postgres
+    AMY_ENABLE_PYDATA=true AMY_PYDATA_USERNAME=username AMY_PYDATA_PASSWORD=password
+    DATABASE_URL="postgres://postgres:@localhost/testdb"
 
 install:
   - pip install -r requirements.txt
   - pip install coveralls
+  - pip install psycopg2
 
 before_script:
+  - psql -c "CREATE DATABASE testdb;" -U postgres
   - if [[ $CHECK_MIGRATION == true ]]; then
       python manage.py makemigrations --dry-run -e;
       export STATUS_CODE=$?;

amy/settings.py

@@ -15,6 +15,7 @@
 
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.translation import ugettext_lazy as _
+import dj_database_url
 
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 
@@ -192,28 +193,36 @@
 # Database
 # https://docs.djangoproject.com/en/1.7/ref/settings/#databases
 
-if DEBUG:
-    DB_FILENAME = os.environ.get('AMY_DB_FILENAME', 'db.sqlite3')
-else:
-    try:
-        DB_FILENAME = os.environ['AMY_DB_FILENAME']
-    except KeyError as ex:
-        raise ImproperlyConfigured(
-            'You must specify AMY_DB_FILENAME environment variable '
-            'when DEBUG is False.') from ex
+if os.environ.get('AMY_DB_FILENAME') is not None:
+    raise ImproperlyConfigured(
+        'AMY_DB_FILENAME environment variable is not used any longer. '
+        'Use DATABASE_URL instead.'
+    )
 
+# By default, local infile db.sqlite3 database is used. If you want to use
+# another sqlite3 database, set env var:
+#
+# $ export DATABASE_URL='sqlite://another-db.sqlite3'
+#
+# or use PostgreSQL database:
+#
+# $ export DATABASE_URL=postgres://username:password@localhost/database_name
+
+DEFAULT_DATABASE_URL = 'sqlite://db.sqlite3'
+db_from_env = dj_database_url.config(
+    conn_max_age=500,
+    default=DEFAULT_DATABASE_URL,
+)
 DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, DB_FILENAME),
-        'TEST': {},
-    }
+    'default': db_from_env,
 }
-if '--keepdb' in sys.argv:
+
+if 'sqlite3' in DATABASES['default']['ENGINE'] and '--keepdb' in sys.argv:
     # By default, Django uses in-memory sqlite3 database, which is much
     # faster than sqlite3 database in a file. However, we may want to keep
     # database between test launches, so that we avoid the overhead of
     # applying migrations on each test launch.
+    DATABASES['default'].setdefault('TEST', {})
     DATABASES['default']['TEST']['NAME'] = 'test_db.sqlite3'
 
 # Authentication

api/views.py

@@ -1,7 +1,17 @@
 import datetime
 from itertools import accumulate
 
-from django.db.models import Count, Sum, Case, F, When, Value, IntegerField, Min
+from django.db.models import (
+    Count, 
+    Sum, 
+    Case, 
+    F, 
+    When, 
+    Value, 
+    IntegerField, 
+    Min, 
+    Prefetch,
+)
 from rest_framework import viewsets
 from rest_framework.decorators import list_route
 from rest_framework.filters import DjangoFilterBackend
@@ -150,8 +160,11 @@ class ExportInstructorLocationsView(ListAPIView):
     permission_classes = (IsAuthenticatedOrReadOnly, )
     paginator = None  # disable pagination
 
-    queryset = Airport.objects.exclude(person=None) \
-                              .prefetch_related('person_set')
+    queryset = Airport.objects.exclude(person=None).prefetch_related(
+        # Make sure that we sort instructors by id. This is default behaviour on
+        # SQLite, but not in PostgreSQL. This is necessary to pass
+        # workshops.test_export.TestExportingInstructors.test_serialization.
+        Prefetch('person_set', queryset=Person.objects.order_by('id')))
     serializer_class = ExportInstructorLocationsSerializer
 
 
@@ -518,7 +531,9 @@ def instructors_by_time_queryset(self, start, end):
             event__end__lte=end,
             role__name='instructor',
             person__may_contact=True,
-        ).exclude(event__tags=tags).order_by('event', 'person', 'role') \
+        # Below, we need to use event__tags__in instead of event__tags,
+        # otherwise it won't work on PostgreSQL.
+        ).exclude(event__tags__in=tags).order_by('event', 'person', 'role') \
          .select_related('person', 'event', 'role')
         return tasks
 

requirements.txt

@@ -22,3 +22,4 @@ django-app-namespace-template-loader>=0.4
 django-webtest==1.8.0
 django-debug-toolbar==1.5
 django-extensions>=1.7,<1.8
+dj-database-url>=0.4,<0.5

workshops/management/commands/instructors_activity.py

@@ -3,9 +3,10 @@
 
 from django.core.management.base import BaseCommand
 from django.core.mail import send_mail
+from django.db.models import Prefetch
 from django.template.loader import get_template
 
-from workshops.models import Badge, Person, Role
+from workshops.models import Badge, Person, Role, Task, Lesson, Award
 
 logger = logging.getLogger()
 
@@ -50,6 +51,7 @@ def fetch_activity(self, may_contact_only=True):
         instructors = instructors.exclude(email__isnull=True)
         if may_contact_only:
             instructors = instructors.exclude(may_contact=False)
+        instructors = instructors.order_by('id')
 
         # let's get some things faster
         instructors = instructors.select_related('airport') \

workshops/migrations/0097_auto_20160519_0739.py

@@ -7,6 +7,9 @@
 from django.db import migrations, models
 
 
+NAME_MAX_LENGTH = 40
+
+
 def populate_languages(apps, schema_editor):
     """Populate the Languages table.
 
@@ -24,7 +27,7 @@ def populate_languages(apps, schema_editor):
             # skip others until we need them
             # https://github.com/swcarpentry/amy/issues/582#issuecomment-159506884
             Language.objects.get_or_create(
-                name=' '.join(language['Description']),
+                name=' '.join(language['Description'])[:NAME_MAX_LENGTH],
                 subtag=language['Subtag']
             )
 
@@ -40,7 +43,7 @@ class Migration(migrations.Migration):
             name='Language',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.CharField(help_text='Description of this language tag in English', max_length=40)),
+                ('name', models.CharField(help_text='Description of this language tag in English', max_length=NAME_MAX_LENGTH)),
                 ('subtag', models.CharField(help_text='Primary language subtag.  https://tools.ietf.org/html/rfc5646#section-2.2.1', max_length=10)),
             ],
         ),

workshops/migrations/0098_auto_20160520_0626.py

@@ -14,11 +14,11 @@ def migrate_language(apps, schema_editor):
     english = Language.objects.get(name='English')
     for request in EventRequest.objects.all():
         # Get the most precisely matching languages
-        language = Language.objects.filter(name__icontains=request.language)\
-                .order_by(Length('name')-len(request.language)).first()
+        language = Language.objects.filter(name__icontains=request.language_old)\
+                .order_by(Length('name')-len(request.language_old)).first()
         if not language:
             language = english
-        request.language_new = language
+        request.language = language
         request.save()
 
 
@@ -29,19 +29,19 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.RenameField(
+            model_name='eventrequest',
+            old_name='language',
+            new_name='language_old',
+        ),
         migrations.AddField(
             model_name='eventrequest',
-            name='language_new',
+            name='language',
             field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='workshops.Language', verbose_name='What human language do you want the workshop to be run in?'),
         ),
         migrations.RunPython(migrate_language),
         migrations.RemoveField(
             model_name='eventrequest',
-            name='language',
-        ),
-        migrations.RenameField(
-            model_name='eventrequest',
-            old_name='language_new',
-            new_name='language',
+            name='language_old',
         ),
     ]

workshops/migrations/0127_fix_language_names.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.12 on 2017-04-26 13:06
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+PREVIOUS_NAME_MAX_LENGTH = 40
+
+
+def fix_truncated_language_names(apps, schema_editor):
+    """Some languages names were truncated in 0097_auto_20160519_0739 migration.
+
+    See https://github.com/swcarpentry/amy/issues/1165 for more info."""
+
+    Language = apps.get_model('workshops', 'Language')
+    languages_to_fix = [
+        'Church Slavic Church Slavonic Old Bulgarian Old Church Slavonic Old Slavonic',
+        'Interlingua (International Auxiliary Language Association)',
+    ]
+    for language_name in languages_to_fix:
+        truncated = language_name[:PREVIOUS_NAME_MAX_LENGTH]
+        try:
+            lang = Language.objects.get(name=truncated)
+        except Language.DoesNotExist:
+            pass
+        else:
+            lang.name = language_name
+            lang.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('workshops', '0126_auto_20170325_0406'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='language',
+            name='name',
+            field=models.CharField(help_text='Description of this language tag in English', max_length=100),
+        ),
+        migrations.RunPython(fix_truncated_language_names),
+    ]

workshops/migrations/0128_merge.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.13 on 2017-07-10 11:19
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('workshops', '0127_make_family_name_optional'),
+        ('workshops', '0127_fix_language_names'),
+    ]
+
+    operations = [
+    ]

workshops/models.py

@@ -738,7 +738,7 @@ class Language(models.Model):
     https://tools.ietf.org/html/rfc5646
     """
     name = models.CharField(
-        max_length=STR_MED,
+        max_length=STR_LONG,
         help_text='Description of this language tag in English')
     subtag = models.CharField(
         max_length=STR_SHORT,

workshops/test/base.py

@@ -77,12 +77,12 @@ def _setUpOrganizations(self):
 
         self.org_alpha = Organization.objects.create(domain='alpha.edu',
                                               fullname='Alpha Organization',
-                                              country='Azerbaijan',
+                                              country='AZ',  # AZ for Azerbaijan
                                               notes='')
 
         self.org_beta = Organization.objects.create(domain='beta.com',
                                              fullname='Beta Organization',
-                                             country='Brazil',
+                                             country='BR',  # BR for Brazil
                                              notes='Notes\nabout\nBrazil\n')
 
     def _setUpAirports(self):

workshops/test/test_commands.py

@@ -218,7 +218,7 @@ def test_getting_events(self):
         Event.objects.all().update(start=date.today())
 
         # one active event with URL and one without
-        e1, e2 = Event.objects.all()[0:2]
+        e1, e2 = Event.objects.order_by('id')[0:2]
         e1.completed = False  # completed == !active
         e1.url = 'https://swcarpentry.github.io/workshop-template/'
         e1.save()
@@ -227,7 +227,7 @@ def test_getting_events(self):
         e2.save()
 
         # one inactive event with URL and one without
-        e3, e4 = Event.objects.all()[2:4]
+        e3, e4 = Event.objects.order_by('id')[2:4]
         e3.completed = True
         e3.url = 'https://datacarpentry.github.io/workshop-template/'
         e3.save()
@@ -236,7 +236,7 @@ def test_getting_events(self):
         e4.save()
 
         # both active but one very old
-        e5, e6 = Event.objects.all()[4:6]
+        e5, e6 = Event.objects.order_by('id')[4:6]
         e5.completed = False
         e5.url = 'https://swcarpentry.github.io/workshop-template2/'
         e5.start = date(2014, 1, 1)