[Fixes #4447] Database locked when using sqlite/spatialite databases

.travis.yml

@@ -207,7 +207,7 @@ install:
       sudo apt-get install -y python-virtualenv python-imaging python-lxml python-pyproj python-shapely python-httplib2 python-httplib2 gettext;
       sudo apt-get install -y python-dev libxml2 libxml2-dev libxslt1-dev zlib1g-dev libjpeg-dev libpq-dev libgdal-dev git default-jdk;
       sudo apt-add-repository -y ppa:jonathonf/backports;
-      sudo apt-get -y update && sudo apt-get install -y sqlite3;
+      sudo apt-get -y update && sudo apt-get install -y sqlite3 spatialite-bin libsqlite3-mod-spatialite;
       sudo apt install -y openjdk-8-jre openjdk-8-jdk ant maven;
       sudo update-java-alternatives --set java-1.8.0-openjdk-amd64;
       export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::");

geonode/geoserver/helpers.py

@@ -1492,24 +1492,6 @@ def ensure_valid_configuration(self, alias):
         except KeyError:
             raise ServerDoesNotExist("The server %s doesn't exist" % alias)
 
-        datastore = server.get('DATASTORE')
-        uploader_backend = getattr(
-            settings,
-            'UPLOADER',
-            dict()).get(
-            'BACKEND',
-            'geonode.rest')
-
-        if uploader_backend == 'geonode.importer' and datastore and not settings.DATABASES.get(
-                datastore):
-            raise ImproperlyConfigured(
-                'The OGC_SERVER setting specifies a datastore '
-                'but no connection parameters are present.')
-
-        if uploader_backend == 'geonode.importer' and not datastore:
-            raise ImproperlyConfigured(
-                'The UPLOADER BACKEND is set to geonode.importer but no DATASTORE is specified.')
-
         if 'PRINTNG_ENABLED' in server:
             raise ImproperlyConfigured("The PRINTNG_ENABLED setting has been removed, use 'PRINT_NG_ENABLED' instead.")
 

geonode/geoserver/tests.py

@@ -32,7 +32,6 @@
 from django.core.urlresolvers import reverse
 from django.contrib.auth import get_user_model
 from django.test.utils import override_settings
-from django.core.exceptions import ImproperlyConfigured
 
 from guardian.shortcuts import assign_perm, remove_perm
 
@@ -1024,31 +1023,24 @@ def test_ogc_server_defaults(self):
 
     @on_ogc_backend(geoserver.BACKEND_PACKAGE)
     def test_importer_configuration(self):
-        """
-        Tests that the OGC_Servers_Handler throws an ImproperlyConfigured exception when using the importer
-        backend without a vector database and a datastore configured.
-        """
         database_settings = self.DATABASE_DEFAULT_SETTINGS.copy()
         ogc_server_settings = self.OGC_DEFAULT_SETTINGS.copy()
         uploader_settings = self.UPLOADER_DEFAULT_SETTINGS.copy()
 
         uploader_settings['BACKEND'] = 'geonode.importer'
         self.assertTrue(['geonode_imports' not in database_settings.keys()])
 
+        # Test the importer backend without specifying a datastore or
+        # corresponding database.
         with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings):
-
-            # Test the importer backend without specifying a datastore or
-            # corresponding database.
-            with self.assertRaises(ImproperlyConfigured):
-                OGC_Servers_Handler(ogc_server_settings)['default']
+            OGC_Servers_Handler(ogc_server_settings)['default']
 
         ogc_server_settings['default']['DATASTORE'] = 'geonode_imports'
 
         # Test the importer backend with a datastore but no corresponding
         # database.
         with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings):
-            with self.assertRaises(ImproperlyConfigured):
-                OGC_Servers_Handler(ogc_server_settings)['default']
+            OGC_Servers_Handler(ogc_server_settings)['default']
 
         database_settings['geonode_imports'] = database_settings[
             'default'].copy()

geonode/layers/utils.py

@@ -941,7 +941,15 @@ def create_thumbnail(instance, thumbnail_remote_url, thumbnail_create_url=None,
                     for _p in params.keys():
                         if _p.lower() not in thumbnail_create_url.lower():
                             thumbnail_create_url = thumbnail_create_url + '&%s=%s' % (_p, params[_p])
-                    resp, image = ogc_client.request(thumbnail_create_url)
+                    headers = {}
+                    if check_ogc_backend(geoserver.BACKEND_PACKAGE):
+                        _user = ogc_server_settings['USER'] if 'USER' in ogc_server_settings else 'admin'
+                        _pwd = ogc_server_settings['PASSWORD'] if 'PASSWORD' in ogc_server_settings else 'geoserver'
+                        import base64
+                        valid_uname_pw = base64.b64encode(
+                            b"%s:%s" % (_user, _pwd)).decode("ascii")
+                        headers['Authorization'] = 'Basic {}'.format(valid_uname_pw)
+                    resp, image = ogc_client.request(thumbnail_create_url, headers=headers)
                     if 'ServiceException' in image or \
                        resp.status_code < 200 or resp.status_code > 299:
                         msg = 'Unable to obtain thumbnail: %s' % image

geonode/settings.py

@@ -100,7 +100,8 @@
 # see https://docs.djangoproject.com/en/1.8/ref/contrib/gis/db-api/#module-django.contrib.gis.db.backends for
 # detailed list of supported backends and notes.
 _db_conf = dj_database_url.parse(DATABASE_URL, conn_max_age=600)
-_db_conf.update({'TIMEOUT': 60})
+if 'spatialite' in DATABASE_URL:
+    SPATIALITE_LIBRARY_PATH = 'mod_spatialite.so'
 DATABASES = {
     'default': _db_conf
 }
@@ -550,7 +551,7 @@
 MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage'
 
 # Security stuff
-SESSION_EXPIRED_CONTROL_ENABLED = ast.literal_eval(os.environ.get('SESSION_EXPIRED_CONTROL_ENABLED', 'False'))
+SESSION_EXPIRED_CONTROL_ENABLED = ast.literal_eval(os.environ.get('SESSION_EXPIRED_CONTROL_ENABLED', 'True'))
 
 if SESSION_EXPIRED_CONTROL_ENABLED:
     # This middleware checks for ACCESS_TOKEN validity and if expired forces
@@ -741,13 +742,13 @@
 )
 
 GEOSERVER_PUBLIC_PORT = os.getenv(
-    'GEOSERVER_PUBLIC_PORT', 8000
+    'GEOSERVER_PUBLIC_PORT', 8080
 )
 
-_default_public_location = '{}://{}:{}/gs/'.format(
+_default_public_location = '{}://{}:{}/geoserver/'.format(
     GEOSERVER_PUBLIC_SCHEMA,
     GEOSERVER_PUBLIC_HOST,
-    GEOSERVER_PUBLIC_PORT) if GEOSERVER_PUBLIC_PORT else '{}://{}/gs/'.format(GEOSERVER_PUBLIC_SCHEMA, GEOSERVER_PUBLIC_HOST)
+    GEOSERVER_PUBLIC_PORT) if GEOSERVER_PUBLIC_PORT else '{}://{}/geoserver/'.format(GEOSERVER_PUBLIC_SCHEMA, GEOSERVER_PUBLIC_HOST)
 
 GEOSERVER_PUBLIC_LOCATION = os.getenv(
     'GEOSERVER_PUBLIC_LOCATION', _default_public_location
@@ -807,9 +808,11 @@
 
 # Uploader Settings
 DATA_UPLOAD_MAX_NUMBER_FIELDS = 100000
+"""
+    DEFAULT_BACKEND_UPLOADER = {'geonode.rest', 'geonode.importer'}
+"""
 UPLOADER = {
     'BACKEND': os.getenv('DEFAULT_BACKEND_UPLOADER', 'geonode.rest'),
-    # 'BACKEND': 'geonode.importer',
     'OPTIONS': {
         'TIME_ENABLED': ast.literal_eval(os.getenv('TIME_ENABLED', 'False')),
         'MOSAIC_ENABLED': ast.literal_eval(os.getenv('MOSAIC_ENABLED', 'False')),

geonode/utils.py

@@ -1279,15 +1279,17 @@ def request(self, url, method='GET', data=None, headers={}, stream=False, timeou
             valid_uname_pw = base64.b64encode(
                 b"%s:%s" % (self.username, self.password)).decode("ascii")
             headers['Authorization'] = 'Basic {}'.format(valid_uname_pw)
-            try:
-                _u = user or get_user_model().objects.get(username=self.username)
-                access_token = get_or_create_token(_u)
-                if access_token and not access_token.is_expired():
-                    headers['Authorization'] = 'Bearer %s' % access_token.token
-            except BaseException:
-                tb = traceback.format_exc()
-                logger.debug(tb)
-                pass
+            if self.username != 'admin' and \
+            connection.cursor().db.vendor not in ('sqlite', 'sqlite3', 'spatialite'):
+                try:
+                    _u = user or get_user_model().objects.get(username=self.username)
+                    access_token = get_or_create_token(_u)
+                    if access_token and not access_token.is_expired():
+                        headers['Authorization'] = 'Bearer %s' % access_token.token
+                except BaseException:
+                    tb = traceback.format_exc()
+                    logger.debug(tb)
+                    pass
 
         response = None
         content = None