Backporting Master PRs

.travis.yml

@@ -2,6 +2,13 @@ sudo: required
 
 language: python
 
+cache:
+  directories:
+    - $HOME/.cache/pip
+
+before_cache:
+  - rm -f $HOME/.cache/pip/log/debug.log
+
 python:
   - "2.7"
 

Dockerfile

@@ -2,7 +2,11 @@ FROM geonode/geonode:latest
 MAINTAINER GeoNode development team
 
 COPY requirements.txt /usr/src/app/
-RUN pip install -r requirements.txt --no-deps
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+RUN python manage.py makemigrations --settings=geonode.settings
+RUN python manage.py migrate --settings=geonode.settings
 
 EXPOSE 8000
-CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
+# CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
+CMD ["paver", "start_django", "-b", "0.0.0.0:8000"]

docker-compose.override.yml

@@ -11,24 +11,12 @@ services:
     environment:
       - DEBUG=True
 
-  celery:
-    build: .
-    # Loading the app is defined here to allow for
-    # autoreload on changes it is mounted on top of the
-    # old copy that docker added when creating the image
-    volumes:
-      - '.:/usr/src/app'
-    environment:
-      - DEBUG=True
-
-
-  consumers:
-    build: .
-    # Loading the app is defined here to allow for
-    # autoreload on changes it is mounted on top of the
-    # old copy that docker added when creating the image
-    volumes:
-      - '.:/usr/src/app'
-    environment:
-      - DEBUG=True 
-
+  # celery:
+  #   build: .
+  #   # Loading the app is defined here to allow for
+  #   # autoreload on changes it is mounted on top of the
+  #   # old copy that docker added when creating the image
+  #   volumes:
+  #     - '.:/usr/src/app'
+  #   environment:
+  #     - DEBUG=True

docker-compose.yml

@@ -16,15 +16,15 @@ services:
     env_file:
       - ./scripts/docker/env/production/db.env
 
-  elasticsearch:
-    image: elasticsearch
-    restart: unless-stopped
-    container_name: elasticsearch4${COMPOSE_PROJECT_NAME}
-    stdin_open: true
-    # tty: true
-    labels:
-        org.geonode.component: elasticsearch
-        org.geonode.instance.name: geonode
+  # elasticsearch:
+  #   image: elasticsearch
+  #   restart: unless-stopped
+  #   container_name: elasticsearch4${COMPOSE_PROJECT_NAME}
+  #   stdin_open: true
+  #   # tty: true
+  #   labels:
+  #       org.geonode.component: elasticsearch
+  #       org.geonode.instance.name: geonode
 
   rabbitmq:
     image: rabbitmq
@@ -36,25 +36,6 @@ services:
         org.geonode.component: rabbitmq
         org.geonode.instance.name: geonode
 
-  django:
-    restart: unless-stopped
-    image: geonode/geonode:latest
-    container_name: django4${COMPOSE_PROJECT_NAME}
-    stdin_open: true
-    # tty: true
-    labels:
-        org.geonode.component: django
-        org.geonode.instance.name: geonode
-    depends_on:
-      - db
-      - elasticsearch
-      - rabbitmq
-    command: django-admin.py runserver 0.0.0.0:8000 --settings=geonode.settings
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    env_file:
-      - ./scripts/docker/env/production/django.env
-
   celery:
     restart: unless-stopped
     image: geonode/geonode:latest
@@ -66,49 +47,52 @@ services:
         org.geonode.instance.name: geonode
     depends_on:
       - db
-      - elasticsearch
+  #     - elasticsearch
       - rabbitmq
     command: celery worker --app=geonode.celery_app:app -B -l INFO
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     env_file:
       - ./scripts/docker/env/production/celery.env
 
-  consumers:
-    image: geonode/geonode:latest
+  geoserver:
+    image: geonode/geoserver:2.13.x
     restart: unless-stopped
-    container_name: consumers4${COMPOSE_PROJECT_NAME}
+    container_name: geoserver4${COMPOSE_PROJECT_NAME}
     stdin_open: true
     # tty: true
     labels:
-        org.geonode.component: consumers
+        org.geonode.component: geoserver
         org.geonode.instance.name: geonode
     depends_on:
-      - rabbitmq
       - db
-      - elasticsearch
-    command: python manage.py runmessaging
+      # - elasticsearch
+      - rabbitmq
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/django.env
+      - ./scripts/docker/env/production/geoserver.env
 
-  geoserver:
-    image: geonode/geoserver:2.13.x
+  django:
     restart: unless-stopped
-    container_name: geoserver4${COMPOSE_PROJECT_NAME}
+    image: geonode/geonode:latest
+    container_name: django4${COMPOSE_PROJECT_NAME}
     stdin_open: true
     # tty: true
     labels:
-        org.geonode.component: geoserver
+        org.geonode.component: django
         org.geonode.instance.name: geonode
     depends_on:
       - db
+      # - elasticsearch
+      - rabbitmq
+    command: paver start_django -b 0.0.0.0:8000
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/geoserver.env
+      - ./scripts/docker/env/production/django.env
 
   geonode:
     image: geonode/nginx:geoserver
@@ -121,6 +105,7 @@ services:
         org.geonode.instance.name: geonode
     depends_on:
       - django
+      - celery
       - geoserver
     ports:
       - "80:80"
@@ -137,7 +122,6 @@ services:
       - geoserver-data-dir:/geoserver_data/data
 
 volumes:
-
   geoserver-data-dir:
     name: ${COMPOSE_PROJECT_NAME}-gsdatadir
   dbdata:

geonode/apps.py

@@ -22,9 +22,10 @@
 
 def run_setup_hooks(*args, **kwargs):
     from django.conf import settings
-    from .celery_app import app as celery_app
+    # from .celery_app import app as celery_app
     if 'celery_app' not in settings.INSTALLED_APPS:
-        settings.INSTALLED_APPS += (celery_app, )
+        # settings.INSTALLED_APPS += (celery_app, )
+        settings.INSTALLED_APPS += ('celery_app', )
 
 
 class AppConfig(BaseAppConfig):

geonode/base/management/commands/backup.py

@@ -345,7 +345,14 @@ def handle(self, **options):
                     print "Saved Static Files from '"+static_files_folder+"'."
 
                 # Store Template Folders
-                template_folders = settings.TEMPLATE_DIRS
+                template_folders = []
+                try:
+                    template_folders = settings.TEMPLATE_DIRS
+                except:
+                    try:
+                        template_folders = settings.TEMPLATES[0]['DIRS']
+                    except:
+                        pass
                 template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
                 if not os.path.exists(template_files_folders):
                     os.makedirs(template_files_folders)

geonode/base/management/commands/restore.py

@@ -288,7 +288,14 @@ def handle(self, **options):
                 static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
                 static_folders = settings.STATICFILES_DIRS
                 static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
-                template_folders = settings.TEMPLATE_DIRS
+                template_folders = []
+                try:
+                    template_folders = settings.TEMPLATE_DIRS
+                except:
+                    try:
+                        template_folders = settings.TEMPLATES[0]['DIRS']
+                    except:
+                        pass
                 template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
                 locale_folders = settings.LOCALE_PATHS
                 locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)

geonode/base/management/commands/settings.ini

@@ -14,8 +14,8 @@ dumprasterdata = yes
 # dumps  = people,accounts,avatars,backups,licenses,topiccategories,regions,spatialrepresentationtypes,restrictioncodetypes,resourcebases,contactroles,links,useropermissions,groupopermissions,uploadsessions,styles,layers,attributes,layerfiles,maps,maplayers,mapsnapshots,documents,tags
 
 # 2.6+
-apps  = contenttypes,auth,people,groups,account,actstream,admin,agon_ratings,announcements,avatar,base,dialogos,documents,guardian,invitations,layers,maps,oauth2_provider,pinax_notifications,services,sessions,sites,taggit,tastypie,upload,user_messages
-dumps = contenttypes,auth,people,groups,account,actstream,admin,agon_ratings,announcements,avatar,base,dialogos,documents,guardian,invitations,layers,maps,oauth2_provider,pinax_notifications,services,sessions,sites,taggit,tastypie,upload,user_messages
+apps  = contenttypes,auth,people,groups,account,guardian,admin,actstream,agon_ratings,announcements,avatar,base,dialogos,documents,geoserver,invitations,layers,maps,oauth2_provider,pinax_notifications,services,sites,socialaccount,taggit,tastypie,upload,user_messages
+dumps = contenttypes,auth,people,groups,account,guardian,admin,actstream,agon_ratings,announcements,avatar,base,dialogos,documents,geoserver,invitations,layers,maps,oauth2_provider,pinax_notifications,services,sites,socialaccount,taggit,tastypie,upload,user_messages
 
 # Migrate from GN 2.0 to GN 2.4
 # migrations = base.resourcebase,layers.layer,layers.attribute,maps.map,maps.maplayer

geonode/base/management/commands/updategeoip.py

@@ -22,6 +22,7 @@
 import os
 import logging
 import gzip
+import tarfile
 import urllib2 as urllib
 import traceback
 
@@ -33,7 +34,14 @@
 
 logger = logging.getLogger(__name__)
 
-URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
+try:
+    from django.contrib.gis.geoip2 import GeoIP2 as GeoIP
+    URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz'
+    OLD_FORMAT = False
+except ImportError:
+    from django.contrib.gis.geoip import GeoIP
+    URL = 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
+    OLD_FORMAT = True
 
 
 class Command(BaseCommand):
@@ -58,6 +66,37 @@ def handle(self, *args, **options):
         logger.info("Requesting %s", options['url'])
 
         r = urllib.urlopen(options['url'], timeout=10.0)
+
+        if OLD_FORMAT:
+            self.handle_old_format(r, fname)
+        else:
+            self.handle_new_format(r, fname)
+
+
+    def handle_new_format(self, r, fname):
+        try:
+            data = StringIO(r.read())
+            with tarfile.open(fileobj=data) as zfile:
+                members = zfile.getmembers()
+                for m in members:
+                    if m.name.endswith('GeoLite2-City.mmdb'):
+                        with open(fname, 'wb') as tofile:
+                            try:
+                                fromfile = zfile.extractfile(m)
+                                logger.info("Writing to %s", fname)
+                                tofile.write(fromfile.read())
+                            except Exception, err:
+                                logger.error("Cannot extract %s and write to %s: %s", m, fname, err, exc_info=err) 
+                                try:
+                                    os.remove(fname)
+                                except OSError:
+                                    logger.debug("Could not delete file %s", fname)
+                        return
+        except Exception, err:
+            logger.error("Cannot process %s: %s", r, err, exc_info=err)
+
+
+    def handle_old_format(self, r, fname):
         try:
             data = StringIO(r.read())
             with gzip.GzipFile(fileobj=data) as zfile: