From 7a89442937b8561dda1fe462f763df4ba40cebe5 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 17:36:10 +0200 Subject: [PATCH 01/49] 1.10 unstable --- cartoview/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cartoview/__init__.py b/cartoview/__init__.py index ad89171e..c410bed6 100644 --- a/cartoview/__init__.py +++ b/cartoview/__init__.py @@ -1,4 +1,4 @@ -__version__ = (1, 8, 6, 'unstable', 0) +__version__ = (1, 10, 0, 'unstable', 0) __compatible_with__ = [] diff --git a/setup.py b/setup.py index f39edef0..0d61dbb9 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ], license="BSD", install_requires=[ - 'future', 'six==1.10.0', 'geonode>2.7.5.dev20180125135927,<2.9', + 'future', 'six==1.10.0', 'geonode>2.7.5.dev20180125135927', 'Faker>=0.8.4', 'cherrypy==11.0.0', 'cheroot==5.8.3', 'mkdocs==0.17.5', 'mkdocs-material==3.0.5', 'SQLAlchemy==1.2.13' ]) From 79bd1f6b993be4fa62ea8227c6eb0237788754e6 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 17:38:00 +0200 Subject: [PATCH 02/49] Django 1.11 compatibility [1] --- .../templates/app_manager/app_install_base.html | 1 - .../templates/app_manager/dynamic_scripts.html | 12 ++++++++++-- cartoview/apps_handler/apps.py | 3 +-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cartoview/app_manager/templates/app_manager/app_install_base.html b/cartoview/app_manager/templates/app_manager/app_install_base.html index 3cfce5d5..e99d4623 100644 --- a/cartoview/app_manager/templates/app_manager/app_install_base.html +++ b/cartoview/app_manager/templates/app_manager/app_install_base.html @@ -1,6 +1,5 @@ {% extends "geonode_base.html" %} {% load i18n %} -{% load url from future %} {% block title %} {{ block.super }} {% endblock %} diff --git a/cartoview/app_manager/templates/app_manager/dynamic_scripts.html b/cartoview/app_manager/templates/app_manager/dynamic_scripts.html index 7375a16b..6e753e78 100644 --- a/cartoview/app_manager/templates/app_manager/dynamic_scripts.html +++ b/cartoview/app_manager/templates/app_manager/dynamic_scripts.html @@ -13,8 +13,16 @@ RESTART_SERVER: "{% url 'restart_server' api_name='api' resource_name='app' %}", }); angular.module('cartoview.userInfo',[]).service("cartoviewUser",function () { - this.isAuthenticated = {{ user.is_authenticated | dump_json }}; - this.isSuperUser = {{ user.is_superuser | dump_json }}; + {% if user.is_authenticated %} + this.isAuthenticated = true + {% else %} + this.isAuthenticated = false + {% endif %} + {% if user.is_superuser %} + this.isSuperUser = true + {% else %} + this.isSuperUser = false + {% endif %} this.loginUrl = "{% url 'account_login' %}?next={{ request.get_full_path }}"; this.logoutUrl = "{% url 'account_logout' %}"; this.registerUrl = "{% url 'account_signup' %}"; diff --git a/cartoview/apps_handler/apps.py b/cartoview/apps_handler/apps.py index 2f22c161..1431648c 100644 --- a/cartoview/apps_handler/apps.py +++ b/cartoview/apps_handler/apps.py @@ -38,8 +38,7 @@ def execute_pending(self): except CommandError as e: error = e.message logger.error(error) - if "you cannot selectively sync unmigrated apps"\ - not in error: + if error and "does not have migrations" not in error: self.delete_application_on_fail(app.name) def ready(self): From e924215b8f21614c4f03c866e6ce2499cd3fb6a3 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 17:38:48 +0200 Subject: [PATCH 03/49] fix app instance model compatibility with geonode2.10.x --- .../migrations/0010_auto_20181118_1333.py | 28 +++++++++++++++++++ cartoview/app_manager/models.py | 20 ++++++++++++- cartoview/app_manager/rest.py | 2 +- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 cartoview/app_manager/migrations/0010_auto_20181118_1333.py diff --git a/cartoview/app_manager/migrations/0010_auto_20181118_1333.py b/cartoview/app_manager/migrations/0010_auto_20181118_1333.py new file mode 100644 index 00000000..a26727da --- /dev/null +++ b/cartoview/app_manager/migrations/0010_auto_20181118_1333.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2018-11-18 13:33 +from __future__ import unicode_literals + +from django.db import migrations +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('app_manager', '0009_auto_20180920_1659'), + ] + + operations = [ + migrations.AlterModelManagers( + name='appinstance', + managers=[ + ('objects', django.db.models.manager.Manager()), + ('base_objects', django.db.models.manager.Manager()), + ], + ), + migrations.RenameField( + model_name='appinstance', + old_name='map', + new_name='related_map', + ), + ] diff --git a/cartoview/app_manager/models.py b/cartoview/app_manager/models.py index 1f50abd7..06751ba9 100644 --- a/cartoview/app_manager/models.py +++ b/cartoview/app_manager/models.py @@ -146,7 +146,7 @@ class AppInstance(ResourceBase): app = models.ForeignKey( App, null=True, blank=True, on_delete=models.CASCADE) config = models.TextField(null=True, blank=True) - map = models.ForeignKey( + related_map = models.ForeignKey( GeonodeMap, null=True, blank=True, on_delete=models.CASCADE) logo = models.ImageField( upload_to=get_app_logo_path, blank=True, null=True) @@ -161,6 +161,24 @@ def name_long(self): else: return '%s (%s)' % (self.title, self.id) + # NOTE:backward compatibility for old map field use by apps \ + # in StandardAppViews + @property + def map_id(self): + return self.related_map_id + + @map_id.setter + def map_id(self, value): + self.related_map_id = value + + @property + def map(self): + return self.related_map + + @map.setter + def map(self, value): + self.rerelated_map = value + @property def config_obj(self): try: diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index 8aeca15a..4be32601 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -294,7 +294,7 @@ class AppInstanceResource(ModelResource): launch_app_url = fields.CharField(null=True, blank=True) edit_url = fields.CharField(null=True, blank=True) app = fields.ForeignKey(AppResource, 'app', full=True, null=True) - map = fields.ForeignKey(MapResource, 'map', full=True, null=True) + map = fields.ForeignKey(MapResource, 'related_map', full=True, null=True) owner = fields.ForeignKey( ProfileResource, 'owner', full=True, null=True, blank=True) keywords = fields.ListField(null=True, blank=True) From faef11c7998afb1c0abd1401a29e74f87f16b47f Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 18:15:43 +0200 Subject: [PATCH 04/49] update local_settings.py --- cartoview/local_settings.py.sample | 364 +++++------------------------ 1 file changed, 58 insertions(+), 306 deletions(-) diff --git a/cartoview/local_settings.py.sample b/cartoview/local_settings.py.sample index f47372b0..04a284f1 100644 --- a/cartoview/local_settings.py.sample +++ b/cartoview/local_settings.py.sample @@ -18,17 +18,19 @@ # ######################################################################### +from past.builtins import execfile +from cartoview import app_manager import os from .settings import (MAP_BASELAYERS, INSTALLED_APPS, OGC_SERVER, TEMPLATES, - CARTOVIEW_TEMPLATE_DIRS, APPS_DIR) + CARTOVIEW_TEMPLATE_DIRS, APPS_DIR, PROJECT_ROOT) SITEURL = "http://localhost/" ALLOWED_HOSTS = ['*'] DATABASES = { 'default': { 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'NAME': 'cartoview', - 'USER': 'xxxx', - 'PASSWORD': 'xxxxx', + 'USER': 'user', + 'PASSWORD': 'password', 'HOST': 'localhost', 'PORT': '5432', }, @@ -37,77 +39,71 @@ DATABASES = { 'ENGINE': 'django.contrib.gis.db.backends.postgis', # 'ENGINE': '', # Empty ENGINE name disables 'NAME': 'cartoview_datastore', - 'USER': 'xxxxxx', - 'PASSWORD': 'xxxxx', + 'USER': 'user', + 'PASSWORD': 'password', 'HOST': 'localhost', 'PORT': '5432', } } GEOSERVER_LOCATION = os.getenv('GEOSERVER_LOCATION', - 'http://localhost:8080/geoserver/') + 'http://localhost:8080/geoserver14-geonode/') GEOSERVER_PUBLIC_LOCATION = os.getenv( - # 'GEOSERVER_PUBLIC_LOCATION', '{}/geoserver/'.format(SITEURL) 'GEOSERVER_LOCATION', - 'http://localhost/geoserver/') + 'http://localhost/geoserver14-geonode/') + +OGC_SERVER_DEFAULT_USER = os.getenv( + 'GEOSERVER_ADMIN_USER', 'admin' +) + +OGC_SERVER_DEFAULT_PASSWORD = os.getenv( + 'GEOSERVER_ADMIN_PASSWORD', 'geoserver' +) + +GEOFENCE_SECURITY_ENABLED = True -# OGC_SERVER_DEFAULT_USER = os.getenv('GEOSERVER_ADMIN_USER', 'admin') +# OGC (WMS/WFS/WCS) Server Settings +# OGC (WMS/WFS/WCS) Server Settings +OGC_SERVER = { + 'default': { + 'BACKEND': 'geonode.geoserver', + 'LOCATION': GEOSERVER_LOCATION, + 'LOGIN_ENDPOINT': 'j_spring_oauth2_geonode_login', + 'LOGOUT_ENDPOINT': 'j_spring_oauth2_geonode_logout', + # PUBLIC_LOCATION needs to be kept like this because in dev mode + # the proxy won't work and the integration tests will fail + # the entire block has to be overridden in the local_settings + 'PUBLIC_LOCATION': GEOSERVER_PUBLIC_LOCATION, + 'USER': OGC_SERVER_DEFAULT_USER, + 'PASSWORD': OGC_SERVER_DEFAULT_PASSWORD, + 'MAPFISH_PRINT_ENABLED': True, + 'PRINT_NG_ENABLED': True, + 'GEONODE_SECURITY_ENABLED': True, + 'GEOFENCE_SECURITY_ENABLED': GEOFENCE_SECURITY_ENABLED, + 'GEOFENCE_URL': os.getenv('GEOFENCE_URL', 'internal:/'), + 'GEOGIG_ENABLED': False, + 'WMST_ENABLED': False, + 'BACKEND_WRITE_ENABLED': True, + 'WPS_ENABLED': False, + 'LOG_FILE': '%s/geoserver/data/logs/geoserver.log' + % os.path.abspath(os.path.join(PROJECT_ROOT, os.pardir)), + # Set to name of database in DATABASES dictionary to enable + # 'datastore', + 'DATASTORE': os.getenv('DEFAULT_BACKEND_DATASTORE', ''), + 'PG_GEOGIG': False, + # 'CACHE': ".cache" # local cache file to for HTTP requests + # number of seconds to allow for HTTP requests + 'TIMEOUT': int(os.getenv('OGC_REQUEST_TIMEOUT', '10')) + } +} -# OGC_SERVER_DEFAULT_PASSWORD = os.getenv('GEOSERVER_ADMIN_PASSWORD', -# 'geoserver') -# OGC_SERVER = { -# 'default': { -# 'BACKEND': -# 'geonode.geoserver', -# 'LOCATION': -# GEOSERVER_LOCATION, -# 'LOGIN_ENDPOINT': -# 'j_spring_oauth2_geonode_login', -# 'LOGOUT_ENDPOINT': -# 'j_spring_oauth2_geonode_logout', -# # PUBLIC_LOCATION needs to be kept like this because in dev mode -# # the proxy won't work and the integration tests will fail -# # the entire block has to be overridden in the local_settings -# 'PUBLIC_LOCATION': -# GEOSERVER_PUBLIC_LOCATION, -# 'USER': -# OGC_SERVER_DEFAULT_USER, -# 'PASSWORD': -# OGC_SERVER_DEFAULT_PASSWORD, -# 'MAPFISH_PRINT_ENABLED': -# True, -# 'PRINT_NG_ENABLED': -# True, -# 'GEONODE_SECURITY_ENABLED': -# True, -# 'GEOFENCE_SECURITY_ENABLED': -# GEOFENCE_SECURITY_ENABLED, -# 'GEOGIG_ENABLED': -# False, -# 'WMST_ENABLED': -# False, -# 'BACKEND_WRITE_ENABLED': -# True, -# 'WPS_ENABLED': -# False, -# 'LOG_FILE': -# '%s/geoserver/data/logs/geoserver.log' % os.path.abspath( -# os.path.join(PROJECT_ROOT, os.pardir)), -# # Set to name of database in DATABASES dictionary to enable -# 'DATASTORE': -# os.getenv('DEFAULT_BACKEND_DATASTORE', 'datastore'), -# 'PG_GEOGIG': -# False, -# 'TIMEOUT': -# 10 # number of seconds to allow for HTTP requests -# } -# } # GeoNode javascript client configuration -# default map projection -# Note: If set to EPSG:4326, then only EPSG:4326 basemaps will work. -DEFAULT_MAP_CRS = "EPSG:900913" + +DEFAULT_MAP_CRS = "EPSG:3857" + +DEFAULT_LAYER_FORMAT = "image/png" # Where should newly created maps be focused? DEFAULT_MAP_CENTER = (0, 0) @@ -116,251 +112,7 @@ DEFAULT_MAP_CENTER = (0, 0) # 0 = entire world; # maximum zoom is between 12 and 15 (for Google Maps, coverage varies by area) DEFAULT_MAP_ZOOM = 0 - -#BASE MAPS -ALT_OSM_BASEMAPS = os.environ.get('ALT_OSM_BASEMAPS', True) -CARTODB_BASEMAPS = os.environ.get('CARTODB_BASEMAPS', True) -STAMEN_BASEMAPS = os.environ.get('STAMEN_BASEMAPS', True) -THUNDERFOREST_BASEMAPS = os.environ.get('THUNDERFOREST_BASEMAPS', True) -MAPBOX_ACCESS_TOKEN = os.environ.get( - 'MAPBOX_ACCESS_TOKEN', - 'pk.eyJ1IjoiaGlzaGFta2FyYW0iLCJhIjoiY2l4dDM5NGU0MDAweDJwbnZma2tvdTExbyJ9.Nyx3DZ15RhW_tl5sPCnpdQ' -) -BING_API_KEY = os.environ.get('BING_API_KEY', '') -GOOGLE_API_KEY = os.environ.get('GOOGLE_API_KEY', '') -MAPBOX_API = { - 'styles': { - 'streets-v9': { - 'enabled': True, - 'name': 'Mapbox Streetmap', - 'attribution': ' Mapbox OpenStreetMap', - 'visibility': False, - }, - 'outdoors-v9': { - 'enabled': True, - 'name': 'Mapbox Outdoors', - 'attribution': ' Mapbox OpenStreetMap', - 'visibility': False, - }, - 'dark-v9': { - 'enabled': True, - 'name': 'Mapbox Dark', - 'attribution': ' Mapbox OpenStreetMap', - 'visibility': False, - }, - 'light-v9': { - 'enabled': True, - 'name': 'Mapbox Light', - 'attribution': ' Mapbox OpenStreetMap', - 'visibility': False, - }, - 'satellite-v9': { - 'enabled': True, - 'name': 'Mapbox Satellite', - 'attribution': ' Mapbox DigitalGlobe', - 'visibility': False, - }, - 'satellite-streets-v9': { - 'enabled': True, - 'name': 'Mapbox Satellite Streets', - 'attribution': ' Mapbox OpenStreetMap DigitalGlobe', - 'visibility': False, - }, - } -} - -MAP_BASELAYERS = [ - { - "source": { - "ptype": "gxp_olsource" - }, - "type": "OpenLayers.Layer", - "args": ["No background"], - "name": "background", - "visibility": False, - "fixed": True, - "group": "background" - }, - # { - # "source": {"ptype": "gxp_olsource"}, - # "type": "OpenLayers.Layer.XYZ", - # "title": "TEST TILE", - # "args": ["TEST_TILE", "http://test_tiles/tiles/${z}/${x}/${y}.png"], - # "name": "background", - # "attribution": "© TEST TILE", - # "visibility": False, - # "fixed": True, - # "group":"background" - # }, - { - "source": { - "ptype": "gxp_osmsource" - }, - "type": "OpenLayers.Layer.OSM", - "name": "mapnik", - "visibility": True, - "fixed": True, - "group": "background" - } -] -for k, v in MAPBOX_API['styles'].items(): - URL = ('https://api.mapbox.com/styles/v1/mapbox/%s/tiles/256/${z}/${x}/' - '${y}?access_token=%s') % (k, MAPBOX_ACCESS_TOKEN) - if v['enabled']: - BASEMAP = { - 'source': { - 'ptype': 'gxp_olsource' - }, - 'type': - 'OpenLayers.Layer.XYZ', - "args": [ - '%s' % v['name'], [URL], - { - 'transitionEffect': 'resize', - 'attribution': '%s' % v['attribution'] - } - ], - 'fixed': - True, - 'visibility': - v['visibility'], - 'group': - 'background' - } - MAP_BASELAYERS.append(BASEMAP) -if 'geonode.geoserver' in INSTALLED_APPS: - LOCAL_GEOSERVER = { - "source": { - "ptype": "gxp_wmscsource", - "url": OGC_SERVER['default']['PUBLIC_LOCATION'] + "wms", - "restUrl": "/gs/rest" - } - } - baselayers = MAP_BASELAYERS - MAP_BASELAYERS = [LOCAL_GEOSERVER] - MAP_BASELAYERS.extend(baselayers) - -OSM = { - 'maps': { - 'de': { - 'enabled': - True, - 'name': - 'OSM DE', - 'visibility': - False, - 'url': ('http://a.tile.openstreetmap.de/tiles/osmde/${z}/${x}/${y}' - '.png'), - 'attribution': ('© OpenStreetMap') - }, - 'france': { - 'enabled': - True, - 'name': - 'OSM France', - 'visibility': - False, - 'url': ('http://a.tile.openstreetmap.fr/osmfr/${z}/${x}/${y}.png'), - 'attribution': ('© Openstreetmap France | © OpenStreetMa' - 'p') - }, - 'hot': { - 'enabled': - True, - 'name': - 'OSM HOT', - 'visibility': - False, - 'url': ('http://a.tile.openstreetmap.fr/hot/${z}/${x}/${y}.png'), - 'attribution': ('© OpenStreetMap, Tiles courtesy of Humanitarian OpenStreetMap Team') - } - } -} - -for k, v in OSM['maps'].items(): - if v['enabled']: - BASEMAP = { - 'source': { - 'ptype': 'gxp_olsource' - }, - 'type': - 'OpenLayers.Layer.XYZ', - "args": [ - '%s' % v['name'], [v['url']], - { - 'transitionEffect': 'resize', - 'attribution': '%s' % v['attribution'], - } - ], - 'fixed': - True, - 'visibility': - v['visibility'], - 'group': - 'background' - } - MAP_BASELAYERS.append(BASEMAP) -STAMEN = { - 'maps': { - 'toner': { - 'enabled': True, - 'name': 'Stamen Toner', - 'visibility': False, - }, - 'toner-lite': { - 'enabled': True, - 'name': 'Stamen Toner Lite', - 'visibility': False, - }, - 'watercolor': { - 'enabled': True, - 'name': 'Stamen Watercolor', - 'visibility': False, - } - } -} -ATTRIBUTION = ('Map tiles by Stamen Design, CC BY 3.0 — Map data © OpenStreetMap') -for k, v in STAMEN['maps'].items(): - URL = 'http://stamen-tiles-a.a.ssl.fastly.net/%s/${z}/${x}/${y}.png' % k - if v['enabled']: - BASEMAP = { - 'source': { - 'ptype': 'gxp_olsource' - }, - 'type': - 'OpenLayers.Layer.XYZ', - "args": [ - '%s' % v['name'], [URL], - { - 'transitionEffect': 'resize', - 'attribution': '%s' % ATTRIBUTION, - } - ], - 'fixed': - True, - 'visibility': - v['visibility'], - 'group': - 'background' - } - MAP_BASELAYERS.append(BASEMAP) -DEFAULT_FROM_EMAIL = 'cartoview@cartologic.com' -EMAIL_USE_TLS = True -EMAIL_HOST = 'smtp.gmail.com' -EMAIL_PORT = 587 -EMAIL_HOST_USER = 'cartoview@cartologic.com' -EMAIL_HOST_PASSWORD = 'clogic0706' TEMPLATES[0]["DIRS"] = CARTOVIEW_TEMPLATE_DIRS + TEMPLATES[0]["DIRS"] -from cartoview import app_manager -from past.builtins import execfile app_manager_settings = os.path.join( os.path.dirname(app_manager.__file__), "settings.py") execfile(os.path.realpath(app_manager_settings)) @@ -370,4 +122,4 @@ for settings_file in APPS_SETTINGS: try: execfile(settings_file) except Exception as e: - print(e.message) \ No newline at end of file + print(e.message) From 4d1926ebd8a8d283fd83d770ea4e4b30ab97d13f Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 18:58:32 +0200 Subject: [PATCH 05/49] improve encoding and python3 compatibility --- .../management/commands/update_current_apps.py | 1 + cartoview/app_manager/models.py | 4 ++++ cartoview/app_manager/req_installer.py | 7 +++++++ cartoview/apps_handler/apps.py | 5 +++++ cartoview/apps_handler/db_parser.py | 7 +++++++ cartoview/apps_handler/handlers.py | 6 ++++++ cartoview/apps_handler/utils.py | 10 +++++++++- cartoview/cartoview_api/rest.py | 15 ++++++++------- cartoview/cartoview_api/views.py | 7 ++++--- cartoview/settings.py | 1 + cartoview/site_management/admin.py | 3 ++- cartoview/site_management/models.py | 1 + cartoview/site_management/tests.py | 1 + cartoview/store_api/api.py | 3 ++- cartoview/urls.py | 12 ++++++++---- 15 files changed, 66 insertions(+), 17 deletions(-) diff --git a/cartoview/app_manager/management/commands/update_current_apps.py b/cartoview/app_manager/management/commands/update_current_apps.py index 853732c8..6349cdcc 100644 --- a/cartoview/app_manager/management/commands/update_current_apps.py +++ b/cartoview/app_manager/management/commands/update_current_apps.py @@ -1,3 +1,4 @@ +from __future__ import print_function import requests from cartoview.app_manager.installer import AppJson, remove_unwanted from cartoview.app_manager.models import App diff --git a/cartoview/app_manager/models.py b/cartoview/app_manager/models.py index 06751ba9..9f26685e 100644 --- a/cartoview/app_manager/models.py +++ b/cartoview/app_manager/models.py @@ -136,6 +136,7 @@ def get_app_logo_path(instance, filename): ]) +@python_2_unicode_compatible class AppInstance(ResourceBase): """ An App Instance is any kind of App Instance that can be created @@ -154,6 +155,9 @@ class AppInstance(ResourceBase): def get_absolute_url(self): return reverse('appinstance_detail', args=(self.id, )) + def __str__(self): + return self.title + @property def name_long(self): if not self.title: diff --git a/cartoview/app_manager/req_installer.py b/cartoview/app_manager/req_installer.py index 42a5fbbd..d96895eb 100644 --- a/cartoview/app_manager/req_installer.py +++ b/cartoview/app_manager/req_installer.py @@ -1,7 +1,14 @@ # -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, + unicode_literals) + import os from os import R_OK, access +from future import standard_library + +standard_library.install_aliases() + class ReqFileException(Exception): message = "requirement file doesn't exists!" diff --git a/cartoview/apps_handler/apps.py b/cartoview/apps_handler/apps.py index 1431648c..e4df074e 100644 --- a/cartoview/apps_handler/apps.py +++ b/cartoview/apps_handler/apps.py @@ -1,13 +1,18 @@ # -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, + unicode_literals) + import os from django.apps import AppConfig from django.conf import settings from django.core.management import call_command from django.core.management.base import CommandError +from future import standard_library from cartoview.log_handler import get_logger +standard_library.install_aliases() pending_yaml = settings.PENDING_APPS logger = get_logger(__name__) diff --git a/cartoview/apps_handler/db_parser.py b/cartoview/apps_handler/db_parser.py index 96dee706..d621a7dd 100644 --- a/cartoview/apps_handler/db_parser.py +++ b/cartoview/apps_handler/db_parser.py @@ -1,7 +1,14 @@ +# -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, + unicode_literals) + from urllib import urlencode import dj_database_url from django.conf import settings +from future import standard_library + +standard_library.install_aliases() class DBParseException(Exception): diff --git a/cartoview/apps_handler/handlers.py b/cartoview/apps_handler/handlers.py index d15a7014..988e2149 100644 --- a/cartoview/apps_handler/handlers.py +++ b/cartoview/apps_handler/handlers.py @@ -1,8 +1,12 @@ # -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, + unicode_literals) + import os from contextlib import contextmanager from django.conf import settings +from future import standard_library from sqlalchemy import Column, Integer, String, create_engine from sqlalchemy.ext.automap import automap_base from sqlalchemy.ext.declarative import declarative_base, declared_attr @@ -10,6 +14,8 @@ from sqlalchemy.sql.expression import func from sqlalchemy.types import Boolean, DateTime +standard_library.install_aliases() + class AppsHandlerDBException(Exception): pass diff --git a/cartoview/apps_handler/utils.py b/cartoview/apps_handler/utils.py index 6d2af449..b854c665 100644 --- a/cartoview/apps_handler/utils.py +++ b/cartoview/apps_handler/utils.py @@ -1,9 +1,17 @@ -from django.conf import settings +# -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, + unicode_literals) + import os +from django.conf import settings +from future import standard_library + from cartoview.app_manager.helpers import (change_path_permission, create_direcotry) +standard_library.install_aliases() + def create_apps_dir(apps_dir=getattr(settings, 'APPS_DIR', None)): if not apps_dir: diff --git a/cartoview/cartoview_api/rest.py b/cartoview/cartoview_api/rest.py index 61835a67..38866ba0 100644 --- a/cartoview/cartoview_api/rest.py +++ b/cartoview/cartoview_api/rest.py @@ -1,13 +1,7 @@ +# -*- coding: utf-8 -*- import json from django.core.urlresolvers import reverse -from geonode.api.api import OwnersResource -from geonode.api.authorization import (GeonodeApiKeyAuthentication, - GeoNodeAuthorization) -from geonode.api.resourcebase_api import MapResource, ResourceBaseResource -from geonode.base.models import ResourceBase -from geonode.layers.models import Attribute, Layer -from geonode.maps.models import MapLayer from tastypie import fields from tastypie.authentication import MultiAuthentication, SessionAuthentication from tastypie.constants import ALL, ALL_WITH_RELATIONS @@ -15,6 +9,13 @@ from cartoview.app_manager.models import App, AppInstance from cartoview.app_manager.rest import LayerFilterExtensionResource +from geonode.api.api import OwnersResource +from geonode.api.authorization import (GeonodeApiKeyAuthentication, + GeoNodeAuthorization) +from geonode.api.resourcebase_api import MapResource, ResourceBaseResource +from geonode.base.models import ResourceBase +from geonode.layers.models import Attribute, Layer +from geonode.maps.models import MapLayer type_filter = { 'app': 'appinstance', diff --git a/cartoview/cartoview_api/views.py b/cartoview/cartoview_api/views.py index 30c11e62..fd03ed49 100644 --- a/cartoview/cartoview_api/views.py +++ b/cartoview/cartoview_api/views.py @@ -1,11 +1,12 @@ +# -*- coding: utf-8 -*- import requests from django.shortcuts import HttpResponse -from geonode.geoserver.helpers import ogc_server_settings -from geonode.layers.views import (_PERMISSION_MSG_MODIFY, _resolve_layer, - layer_detail) from requests.auth import HTTPBasicAuth from cartoview.log_handler import get_logger +from geonode.geoserver.helpers import ogc_server_settings +from geonode.layers.views import (_PERMISSION_MSG_MODIFY, _resolve_layer, + layer_detail) try: import simplejson as json diff --git a/cartoview/settings.py b/cartoview/settings.py index 03825c54..27c33a55 100644 --- a/cartoview/settings.py +++ b/cartoview/settings.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import print_function import ast import copy import os diff --git a/cartoview/site_management/admin.py b/cartoview/site_management/admin.py index f0d85524..95de6b5d 100644 --- a/cartoview/site_management/admin.py +++ b/cartoview/site_management/admin.py @@ -1,6 +1,7 @@ +# -*- coding: utf-8 -*- from django.contrib import admin + from .models import SiteLogo -# Register your models here. @admin.register(SiteLogo) diff --git a/cartoview/site_management/models.py b/cartoview/site_management/models.py index 6188d6ef..a521d112 100644 --- a/cartoview/site_management/models.py +++ b/cartoview/site_management/models.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from datetime import datetime from django.contrib.sites.models import Site diff --git a/cartoview/site_management/tests.py b/cartoview/site_management/tests.py index d420b230..fb545481 100644 --- a/cartoview/site_management/tests.py +++ b/cartoview/site_management/tests.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os from django.conf import settings diff --git a/cartoview/store_api/api.py b/cartoview/store_api/api.py index 7d7ac8ac..32719e50 100644 --- a/cartoview/store_api/api.py +++ b/cartoview/store_api/api.py @@ -1,5 +1,6 @@ -from tastypie.resources import Resource +# -*- coding: utf-8 -*- from tastypie import fields +from tastypie.resources import Resource class GenericApiResource(object): diff --git a/cartoview/urls.py b/cartoview/urls.py index 3bcfe489..366fb302 100644 --- a/cartoview/urls.py +++ b/cartoview/urls.py @@ -1,15 +1,19 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import + from django.conf.urls import include, url -from geonode.api.urls import api -from geonode.urls import urlpatterns as geonode_urls from cartoview.app_manager.rest import (AppInstanceResource, AppResource, AppTypeResource, LayerFilterExtensionResource) from cartoview.cartoview_api.views import layer_config_json, update_extent from cartoview.views import check_version -from cartoview_api.rest import (AllResourcesResource, AttributeResource, - ExtendedResourceBaseResource, MapLayerResource) +from geonode.api.urls import api +from geonode.urls import urlpatterns as geonode_urls + +from .cartoview_api.rest import (AllResourcesResource, AttributeResource, + ExtendedResourceBaseResource, + MapLayerResource) api.register(AppInstanceResource()) api.register(AppResource()) From 715929e998c1834800c0775d7e0bf2e47f6e9e62 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 19:01:38 +0200 Subject: [PATCH 06/49] add requirements.txt --- requirements.txt | 227 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..0f97d8a0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,227 @@ +amqp==2.2.2 +anyjson==0.3.3 +appnope==0.1.0 +arcrest==10.3 +arrow==0.12.1 +asn1crypto==0.24.0 +attrs==17.4.0 +autoflake==0.7 +Automat==0.7.0 +autopep8==1.4.3 +awesome-slugify==1.6.5 +backports-abc==0.5 +backports.functools-lru-cache==1.5 +backports.shutil-get-terminal-size==1.0.0 +backports.ssl-match-hostname==3.5.0.1 +beautifulsoup4==4.4.1 +billiard==3.5.0.3 +boto==2.38.0 +-e git+git@github.com:cartologic/cartoview.git@4d1926ebd8a8d283fd83d770ea4e4b30ab97d13f#egg=cartoview +celery==4.2.1 +certifi==2018.1.18 +cffi==1.11.5 +chardet==3.0.4 +cheroot==5.8.3 +CherryPy==11.0.0 +click==6.7 +configparser==3.5.0 +constantly==15.1.0 +coreapi==2.3.3 +coreschema==0.0.4 +coverage==4.5.1 +cryptography==2.3.1 +datautil==0.4 +decorator==4.1.2 +dicttoxml==1.7.4 +diff-match-patch==20121119 +dj-database-url==0.4.2 +dj-pagination==2.3.2 +Django==1.11.16 +django-activity-stream==0.6.5 +django-allauth==0.34.0 +django-appconf==1.0.2 +django-apptemplates==1.4 +django-autocomplete-light==2.3.3 +django-autofixture==0.12.1 +django-autoslug==1.9.3 +django-basic-authentication-decorator==0.9 +django-bootstrap-form==3.4 +django-bootstrap3-datetimepicker-2==2.5.0 +django-braces==1.12.0 +django-celery-beat==1.1.1 +django-celery-monitor==1.1.2 +django-celery-results==1.0.1 +django-cors-headers==2.2.0 +django-crispy-forms==1.7.2 +django-cuser==2017.3.16 +django-downloadview==1.9 +django-extensions==2.0.7 +django-filter==2.0.0 +django-floppyforms==1.7.0 +django-forms-bootstrap==3.1.0 +django-geoexplorer==4.0.41 +django-geoexplorer-worldmap==4.0.53 +django-geonode-client==1.0.9 +django-guardian==1.4.9 +django-haystack==2.8.1 +django-import-export==1.0.0 +django-invitations==1.9.2 +django-ipware==2.1.0 +django-js-asset==1.0.0 +django-jsonfield==1.0.1 +django-jsonfield-compat==0.4.4 +django-leaflet==0.23.0 +django-model-utils==3.1.1 +django-modeltranslation==0.12.2 +django-mptt==0.9.0 +django-multi-email-field==0.5.1 +django-nose==1.4.5 +django-polymorphic==1.3.1 +django-reversion==2.0.13 +django-simple-history==1.9.0 +django-storages==1.6.5 +django-suit==0.2.26 +django-taggit==0.22.2 +django-tastypie==0.14.0 +django-treebeard==4.2.1 +django-utils==0.0.2 +djangorestframework==3.8.2 +docker==3.1.1 +docker-pycreds==0.3.0 +elasticsearch==2.4.1 +enum34==1.1.6 +et-xmlfile==1.0.1 +factory-boy==2.9.2 +Faker==0.8.4 +flake8==3.5.0 +funcsigs==1.0.2 +future==0.17.1 +futures==3.2.0 +geographiclib==1.49 +geoip2==2.8.0 +geolinks==0.2.0 +-e git+https://github.com/GeoNode/geonode.git@cc010aa61d48da494e03bda0ee75d9b76480fc85#egg=GeoNode +geonode-agon-ratings==0.3.8 +geonode-announcements==1.0.13 +geonode-avatar==2.1.8 +geonode-dialogos==1.2 +geonode-oauth-toolkit==1.1.2rc0 +geonode-user-messages==0.1.14 +geopy==1.14.0 +gisdata==0.5.4 +glob2==0.6 +gn-gsimporter==1.0.9 +gsconfig==1.0.10 +gunicorn==19.7.1 +httplib2==0.10.3 +hyperlink==18.0.0 +idna==2.6 +incremental==17.5.0 +inflection==0.3.1 +invoke==0.22.1 +ipaddress==1.0.18 +ipython==5.5.0 +ipython-genutils==0.2.0 +isort==4.3.4 +itypes==1.1.0 +jaraco.functools==1.20 +jdcal==1.3 +Jinja2==2.10 +jwcrypto==0.6.0 +kombu==4.2.1 +livereload==2.5.2 +lxml==3.6.2 +Mako==1.0.7 +Markdown==2.6.11 +MarkupSafe==1.0 +maxminddb==1.4.1 +mccabe==0.6.1 +mercantile==1.0.4 +mkdocs==0.17.5 +mkdocs-material==3.0.5 +mock==2.0.0 +more-itertools==4.1.0 +MultipartPostHandler==0.1.0 +nose==1.3.7 +numpy==1.13.1 +oauthlib==2.1.0 +odfpy==1.3.6 +openpyxl==2.5.0 +OWSLib==0.16.0 +parse==1.8.2 +parse-type==0.4.1 +pathlib2==2.3.2 +Paver==1.2.4 +pbr==3.1.1 +pep8==1.7.1 +pexpect==4.6.0 +pickleshare==0.7.5 +Pillow==3.3.2 +Pinax==0.9a2 +pinax-notifications==4.1.0 +pinax-theme-bootstrap==8.0.1 +pluggy==0.6.0 +portend==2.3 +poster==0.8.1 +prompt-toolkit==1.0.15 +psutil==5.4.8 +psycopg2==2.7.3.1 +ptyprocess==0.6.0 +py==1.5.3 +pycodestyle==2.4.0 +pycountry==18.5.26 +pycparser==2.18 +pycsw==2.2.0 +pyflakes==1.6.0 +Pygments==2.2.0 +PyHamcrest==1.9.0 +pymdown-extensions==6.0 +pyOpenSSL==18.0.0 +pyproj==1.9.5.1 +pytest==3.5.0 +pytest-bdd==2.20.0 +pytest-django==3.1.2 +pytest-splinter==1.8.5 +python-dateutil==2.6.1 +python-gnupg==0.4.1 +python-memcached==1.59 +python-mimeparse==1.6.0 +python-openid==2.2.5 +pytz==2018.3 +PyYAML==3.12 +regex==2016.7.21 +requests==2.20.1 +requests-oauthlib==1.0.0 +scandir==1.9.0 +selenium==3.9.0 +Shapely==1.5.17 +simplegeneric==0.8.1 +simplejson==3.13.2 +singledispatch==3.4.0.3 +six==1.10.0 +splinter==0.7.7 +SQLAlchemy==1.2.13 +tablib==0.12.1 +tempora==1.14 +timeout-decorator==0.4.0 +tornado==4.5.3 +tqdm==4.23.3 +traitlets==4.3.2 +transifex-client==0.12.4 +Twisted==18.9.0 +typing==3.6.4 +ua-parser==0.8.0 +unicodecsv==0.14.1 +Unidecode==0.4.19 +uritemplate==3.0.0 +urllib3==1.22 +user-agents==1.1.0 +uWSGI==2.0.17 +vine==1.1.4 +wcwidth==0.1.7 +websocket-client==0.54.0 +xlrd==1.1.0 +xlwt==1.3.0 +xmljson==0.1.9 +xmltodict==0.10.2 +zope.interface==4.6.0 From 47b3e54f537f770b64af3b80d33153486f5bedac Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 19:06:34 +0200 Subject: [PATCH 07/49] set app pending to false after migrating app models --- cartoview/apps_handler/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cartoview/apps_handler/apps.py b/cartoview/apps_handler/apps.py index e4df074e..6d5f1b5b 100644 --- a/cartoview/apps_handler/apps.py +++ b/cartoview/apps_handler/apps.py @@ -39,12 +39,13 @@ def execute_pending(self): interactive=False, ignore=['node_modules', '.git']) call_command("migrate", app.name, interactive=False) - CartoApps.set_app_pending(app.name, False) except CommandError as e: error = e.message logger.error(error) if error and "does not have migrations" not in error: self.delete_application_on_fail(app.name) + finally: + CartoApps.set_app_pending(app.name, False) def ready(self): apps_dir = getattr(settings, 'APPS_DIR', None) From 14deef619f91dd3655357bec1d2131266bcf9255 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 19:07:34 +0200 Subject: [PATCH 08/49] fix 'future' is not a registered tag library --- .../app_manager/templates/app_manager/app_instance_list.html | 1 - .../app_manager/templates/app_manager/appinstance_base.html | 1 - .../app_manager/templates/app_manager/appinstance_detail.html | 1 - 3 files changed, 3 deletions(-) diff --git a/cartoview/app_manager/templates/app_manager/app_instance_list.html b/cartoview/app_manager/templates/app_manager/app_instance_list.html index 9c1bde94..e2bd305f 100644 --- a/cartoview/app_manager/templates/app_manager/app_instance_list.html +++ b/cartoview/app_manager/templates/app_manager/app_instance_list.html @@ -1,7 +1,6 @@ {% extends "app_manager/appinstance_base.html" %} {% load i18n %} {% load staticfiles %} -{% load url from future %} {% block title %} {% trans "Explore Apps" %} - {{ block.super }} {% endblock %} {% block head %} diff --git a/cartoview/app_manager/templates/app_manager/appinstance_base.html b/cartoview/app_manager/templates/app_manager/appinstance_base.html index a6c22cdb..8a75db91 100644 --- a/cartoview/app_manager/templates/app_manager/appinstance_base.html +++ b/cartoview/app_manager/templates/app_manager/appinstance_base.html @@ -1,6 +1,5 @@ {% extends "geonode_base.html" %} {% load i18n %} -{% load url from future %} {% block title %} {{ block.super }} {% endblock %} diff --git a/cartoview/app_manager/templates/app_manager/appinstance_detail.html b/cartoview/app_manager/templates/app_manager/appinstance_detail.html index 81996926..1e2450b3 100644 --- a/cartoview/app_manager/templates/app_manager/appinstance_detail.html +++ b/cartoview/app_manager/templates/app_manager/appinstance_detail.html @@ -3,7 +3,6 @@ {% load dialogos_tags %} {% load agon_ratings_tags %} {% load bootstrap_tags %} -{% load url from future %} {% load base_tags %} {% load guardian_tags %} From dd697225dc21d02af9217260213a4e84f1f519d7 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 19:21:40 +0200 Subject: [PATCH 09/49] improve Django 1.11 compatibility --- .../app_manager/appinstance_metadata.html | 1 - cartoview/app_manager/views.py | 41 +++++-------------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/cartoview/app_manager/templates/app_manager/appinstance_metadata.html b/cartoview/app_manager/templates/app_manager/appinstance_metadata.html index 58a0022a..92f1feab 100644 --- a/cartoview/app_manager/templates/app_manager/appinstance_metadata.html +++ b/cartoview/app_manager/templates/app_manager/appinstance_metadata.html @@ -5,7 +5,6 @@ {% block title %}{{ appinstance.typename }} — {{ block.super }}{% endblock %} {% block head %} - {% include "geonode/ext_header.html" %} {{ block.super }} {% endblock %} diff --git a/cartoview/app_manager/views.py b/cartoview/app_manager/views.py index c6cfa795..1cfcd399 100644 --- a/cartoview/app_manager/views.py +++ b/cartoview/app_manager/views.py @@ -18,7 +18,6 @@ from django.forms.utils import ErrorList from django.http import Http404, HttpResponse, HttpResponseRedirect from django.shortcuts import get_object_or_404, render, render_to_response -from django.template import RequestContext, loader from django.utils.decorators import method_decorator from django.utils.translation import ugettext as _ from django.views.decorators.http import require_POST @@ -228,20 +227,13 @@ def appinstance_detail(request, appinstanceid): PERMISSION_MSG_VIEW) except Http404: - return HttpResponse( - loader.render_to_string('404.html', RequestContext(request, {})), - status=404) + return render(request, '404.html', context={}, status=404) except PermissionDenied: - return HttpResponse( - loader.render_to_string( - '401.html', - RequestContext( - request, { + return render(request, '401.html', context={ 'error_message': _("You are not allowed to view this document.") - })), - status=403) + }, status=403) if appinstance is None: return HttpResponse( @@ -279,9 +271,8 @@ def appinstance_detail(request, appinstanceid): context_dict['exif_data'] = exif except BaseException as e: logger.error(e.message + "Exif extraction failed.") - - return render_to_response("app_manager/appinstance_detail.html", - RequestContext(request, context_dict)) + return render(request, "app_manager/appinstance_detail.html", + context=context_dict) @login_required @@ -295,19 +286,11 @@ def appinstance_metadata(request, PERMISSION_MSG_METADATA) except Http404: - return HttpResponse( - loader.render_to_string('404.html', RequestContext(request, {})), - status=404) + return render(request, '404.html', context={}, status=404) except PermissionDenied: - return HttpResponse( - loader.render_to_string( - '401.html', - RequestContext( - request, { - 'error_message': - _("You are not allowed to edit this instance.") - })), + return render(request, '401.html', context={ + 'error_message': _("You are not allowed to edit this instance.")}, status=403) if appinstance is None: @@ -417,17 +400,13 @@ def appinstance_metadata(request, else: author_form = ProfileForm(prefix="author") author_form.hidden = True - - return render_to_response( - template, - RequestContext( - request, { + return render(request, template, context={ "appinstance": appinstance, "appinstance_form": appinstance_form, "poc_form": poc_form, "author_form": author_form, "category_form": category_form, - })) + }) def appinstance_remove(request, appinstanceid): From 6a0294df5bfe5816b4d992d5a6476f481ed5d983 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 18 Nov 2018 19:36:45 +0200 Subject: [PATCH 10/49] remove unused import --- cartoview/app_manager/views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cartoview/app_manager/views.py b/cartoview/app_manager/views.py index 1cfcd399..8abf7410 100644 --- a/cartoview/app_manager/views.py +++ b/cartoview/app_manager/views.py @@ -17,21 +17,21 @@ from django.db.models import F, Max, Min from django.forms.utils import ErrorList from django.http import Http404, HttpResponse, HttpResponseRedirect -from django.shortcuts import get_object_or_404, render, render_to_response +from django.shortcuts import get_object_or_404, render from django.utils.decorators import method_decorator from django.utils.translation import ugettext as _ from django.views.decorators.http import require_POST from future import standard_library from future.utils import with_metaclass +from guardian.shortcuts import get_perms + +from cartoview.app_manager.forms import AppInstanceEditForm +from cartoview.log_handler import get_logger from geonode.base.forms import CategoryForm from geonode.base.models import TopicCategory from geonode.people.forms import ProfileForm from geonode.security.views import _perms_info_json from geonode.utils import build_social_links -from guardian.shortcuts import get_perms - -from cartoview.app_manager.forms import AppInstanceEditForm -from cartoview.log_handler import get_logger from .decorators import (PERMISSION_MSG_DELETE, PERMISSION_MSG_METADATA, PERMISSION_MSG_VIEW, can_change_app_instance, From 05314dfa759512286e322cc7979339874ee5065f Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Mon, 19 Nov 2018 14:23:34 +0200 Subject: [PATCH 11/49] 1.10.0 beta --- .travis.yml | 3 +-- cartoview/__init__.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f45510c7..84be7c79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,10 +70,9 @@ deploy: on: tags: true all_branches: true - condition: $TRAVIS_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+ + condition: $TRAVIS_TAG =~ ^v([0-9]+\.?){2,3}(b[0-9]+|rc[0-9]+){0,1} notifications: email: - hisham.karam@cartologic.com - - hishamwaleedkaram@gmail.com - ahmednosman@cartologic.com - ahmedNourElDeen@cartologic.com diff --git a/cartoview/__init__.py b/cartoview/__init__.py index c410bed6..7099e15c 100644 --- a/cartoview/__init__.py +++ b/cartoview/__init__.py @@ -1,4 +1,4 @@ -__version__ = (1, 10, 0, 'unstable', 0) +__version__ = (1, 10, 0, 'beta', 0) __compatible_with__ = [] From 020d0a6d0bd4add4ff97d2e45669bac6bed280bf Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Mon, 19 Nov 2018 14:51:22 +0200 Subject: [PATCH 12/49] update readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a372c2f3..76fcdc24 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ ## How To Add Cartoview To Existing Geonode: - install cartoview with pip: - - `pip install cartoview==1.8.5 --no-cache-dir` + - `pip install cartoview --no-cache-dir` - open geonode `settings.py` and add the following lines at the end of the file: ```python from cartoview import settings as cartoview_settings From 39d8b046c4d6ca62a35d639843efe045b6e09193 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Wed, 21 Nov 2018 10:36:19 +0200 Subject: [PATCH 13/49] improve geom_type filter --- cartoview/app_manager/rest.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index 4be32601..80b0bcf4 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -54,7 +54,8 @@ def build_filters(self, filters=None, **kwargs): return orm_filters def apply_filters(self, request, applicable_filters): - permission = applicable_filters.pop('permission', None) + permission = applicable_filters.pop( + 'permission', None) # NOTE: We change this filter name from type to geom_type because it # overrides geonode type filter(vector,raster) layer_geom_type = applicable_filters.pop('geom_type', None) @@ -62,18 +63,18 @@ def apply_filters(self, request, applicable_filters): request, applicable_filters) if layer_geom_type: filtered = filtered.filter( - attribute_set__in=Attribute.objects.filter( - attribute_type__icontains=layer_geom_type)) + attribute_set__attribute_type__icontains=layer_geom_type) if permission is not None: - filtered = get_objects_for_user( - request.user, permission, klass=filtered) + permitted_ids = get_objects_for_user( + request.user, permission).values('id') + filtered = filtered.filter(id__in=permitted_ids) return filtered class Meta(LayerResource.Meta): resource_name = "layers" - filtering = dict(LayerResource.Meta.filtering.items() + - {'typename': ALL}.items()) + filtering = dict(LayerResource.Meta.filtering.items() + + {'typename': ALL}.items()) class GeonodeMapLayerResource(ModelResource): From e3467b1e08c98d76224cc7ed74f37ad88a8559e2 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Wed, 21 Nov 2018 10:52:03 +0200 Subject: [PATCH 14/49] fix testing --- cartoview/app_manager/rest.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index 80b0bcf4..b6310d42 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -10,13 +10,6 @@ from django.core.urlresolvers import reverse from django.db import transaction from future import standard_library -from geonode.api.api import ProfileResource -from geonode.api.authorization import GeoNodeAuthorization -from geonode.api.resourcebase_api import (CommonMetaApi, LayerResource, - MapResource) -from geonode.layers.models import Attribute -from geonode.maps.models import MapLayer -from geonode.people.models import Profile from guardian.shortcuts import get_objects_for_user from taggit.models import Tag from tastypie import fields, http @@ -27,6 +20,12 @@ from cartoview.app_manager.models import App, AppInstance, AppStore, AppType from cartoview.apps_handler.handlers import CartoApps from cartoview.log_handler import get_logger +from geonode.api.api import ProfileResource +from geonode.api.authorization import GeoNodeAuthorization +from geonode.api.resourcebase_api import (CommonMetaApi, LayerResource, + MapResource) +from geonode.maps.models import MapLayer +from geonode.people.models import Profile from .installer import AppInstaller, RestartHelper from .resources import FileUploadResource @@ -73,8 +72,7 @@ def apply_filters(self, request, applicable_filters): class Meta(LayerResource.Meta): resource_name = "layers" - filtering = dict(LayerResource.Meta.filtering.items() - + {'typename': ALL}.items()) + filtering = dict(LayerResource.Meta.filtering, **dict(typename=ALL)) class GeonodeMapLayerResource(ModelResource): From 85a3766a93595fc381fd09c80f216b842489a646 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Wed, 21 Nov 2018 11:19:35 +0200 Subject: [PATCH 15/49] fix Cannot determine content type --- cartoview/app_manager/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index b6310d42..928ce427 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -65,7 +65,7 @@ def apply_filters(self, request, applicable_filters): attribute_set__attribute_type__icontains=layer_geom_type) if permission is not None: permitted_ids = get_objects_for_user( - request.user, permission).values('id') + request.user, permission, klass=filtered).values('id') filtered = filtered.filter(id__in=permitted_ids) return filtered From 600922618850fe105d9383f7ce9207996194215e Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Wed, 21 Nov 2018 11:56:22 +0200 Subject: [PATCH 16/49] update required geonode version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0d61dbb9..cbf6f655 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ], license="BSD", install_requires=[ - 'future', 'six==1.10.0', 'geonode>2.7.5.dev20180125135927', + 'future', 'six==1.10.0', 'geonode>=2.10rc4', 'Faker>=0.8.4', 'cherrypy==11.0.0', 'cheroot==5.8.3', 'mkdocs==0.17.5', 'mkdocs-material==3.0.5', 'SQLAlchemy==1.2.13' ]) From 3d3d6f40d65997e7f7f94c8675dbc3fbc8b5531f Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Wed, 21 Nov 2018 12:29:21 +0200 Subject: [PATCH 17/49] fix travis build --- cartoview/app_manager/settings.py | 3 +- cartoview/app_manager/tests.py | 40 ------------------- cartoview/app_manager/tests/test_installer.py | 2 +- cartoview/apps_handler/__init__.py | 2 +- 4 files changed, 3 insertions(+), 44 deletions(-) delete mode 100644 cartoview/app_manager/tests.py diff --git a/cartoview/app_manager/settings.py b/cartoview/app_manager/settings.py index 8f85a066..4fb546c9 100644 --- a/cartoview/app_manager/settings.py +++ b/cartoview/app_manager/settings.py @@ -8,8 +8,6 @@ from future import standard_library from past.builtins import execfile - -from cartoview.apps_handler.handlers import CartoApps, apps_orm from cartoview.apps_handler.utils import create_apps_dir from cartoview.log_handler import get_logger @@ -24,6 +22,7 @@ def load_apps(APPS_DIR): + from cartoview.apps_handler.handlers import CartoApps, apps_orm global CARTOVIEW_APPS global APPS_SETTINGS create_apps_dir(APPS_DIR) diff --git a/cartoview/app_manager/tests.py b/cartoview/app_manager/tests.py deleted file mode 100644 index 84f343ed..00000000 --- a/cartoview/app_manager/tests.py +++ /dev/null @@ -1,40 +0,0 @@ -# from django.test import TestCase -# import mock -# from .models import AppTag, App -# from .config import Item - - -# class AppTagsTestCase(TestCase): - -# def test_AppTag_str(self): -# mock_instance = mock.Mock(spec=AppTag) -# mock_instance.name = "tag" -# mock_instance.__str__ = mock.Mock(return_value=mock_instance.name) -# self.assertEqual(AppTag.__str__(mock_instance), -# mock_instance.__str__()) - -# def test_tag_created(self): -# tag, created = AppTag.objects.get_or_create(name="tag") -# self.assertEqual(created, True) -# self.assertEqual(tag.name, "tag") - - -# class AppTestCase(TestCase): - -# def test_App_str(self): -# app_mock = mock.Mock(spec=App) -# app_mock.name = "cartoview_basic_viewer" -# app_mock.title = "Basic Viewer" -# app_mock.single_instance = False -# app_mock.version = '1.2.3' -# app_mock.settings_url = App.settings_url -# app_mock.__str__ = mock.Mock(return_value=app_mock.title) -# self.assertEqual(App.__str__(app_mock), -# app_mock.__str__()) - -# def test_app_creation(self): -# app, created = App.objects.get_or_create( -# name="cartoview_basic_viewer", title="basic viewer", -# single_instance=False, version='1.2.3') -# self.assertEqual(type(app.config), Item) -# self.assertEqual(created, True) diff --git a/cartoview/app_manager/tests/test_installer.py b/cartoview/app_manager/tests/test_installer.py index 682c6877..8f23e037 100644 --- a/cartoview/app_manager/tests/test_installer.py +++ b/cartoview/app_manager/tests/test_installer.py @@ -22,7 +22,7 @@ def test_app_installer(self): user = Profile.objects.filter(is_superuser=True).first() store_id = 1 app_name = "cartoview_dashboard" - app_version = "1.4.2" + app_version = "1.4.3" app_installer = AppInstaller(app_name, store_id, app_version, user) installed_apps = app_installer.install() self.assertEqual(len(installed_apps), 1) diff --git a/cartoview/apps_handler/__init__.py b/cartoview/apps_handler/__init__.py index 7cd43b8d..aeb72fe3 100644 --- a/cartoview/apps_handler/__init__.py +++ b/cartoview/apps_handler/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from .handlers import apps_orm from .utils import create_apps_dir create_apps_dir() +from .handlers import apps_orm # noqa apps_orm.create_all() From 8bab3786c1bc00b731fd82e366e32e312aceb778 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Thu, 22 Nov 2018 12:17:37 +0200 Subject: [PATCH 18/49] update readme.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 76fcdc24..c65f8d2c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@

+| WARNING: be careful this version(1.10.x) of Cartoview is compatibile with geonode 2.10.x only | +| --- | --- ## What is Cartoview? - CartoView is a GIS Web Mapping Application Market. From 4f515746183c5f1af9a03b218bc9a10380b35c69 Mon Sep 17 00:00:00 2001 From: Hisham waleed karam Date: Thu, 22 Nov 2018 12:31:53 +0200 Subject: [PATCH 19/49] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index c65f8d2c..a3edd90d 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ | WARNING: be careful this version(1.10.x) of Cartoview is compatibile with geonode 2.10.x only | | --- | + --- ## What is Cartoview? - CartoView is a GIS Web Mapping Application Market. @@ -86,3 +87,9 @@ ```sh paver run_test ``` + ___ +## Previous Versions +| Cartoview Version | Geonode Version | docs Link | +|-------------------|-----------------|---------------------------------------------------------------------------------| +| 1.8.x | 2.8.x | [Here](https://github.com/cartologic/cartoview/blob/1.8.x/README.md) | +| 1.6.x | 2.6.x | [Here](https://github.com/cartologic/cartoview/blob/2.6.x_compatible/README.md) | From 236def4c3cc8e5bc41c8342a17fc42c686f5d400 Mon Sep 17 00:00:00 2001 From: Hisham waleed karam Date: Thu, 22 Nov 2018 12:48:19 +0200 Subject: [PATCH 20/49] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a3edd90d..12df7d74 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

-| WARNING: be careful this version(1.10.x) of Cartoview is compatibile with geonode 2.10.x only | +| WARNING: be careful this version(1.10.x) of Cartoview is compatibile with geonode 2.10.x only,if you want to install another version please take alook on [this section](https://github.com/cartologic/cartoview/blob/hisham-dev/README.md#previous-versions) | | --- | --- @@ -88,7 +88,7 @@ paver run_test ``` ___ -## Previous Versions +# Previous Versions | Cartoview Version | Geonode Version | docs Link | |-------------------|-----------------|---------------------------------------------------------------------------------| | 1.8.x | 2.8.x | [Here](https://github.com/cartologic/cartoview/blob/1.8.x/README.md) | From 38b3b67e0461af3465b27386501583542058965d Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 25 Nov 2018 16:51:32 +0200 Subject: [PATCH 21/49] improve how to install app deps --- cartoview/app_manager/decorators.py | 11 +++++++---- cartoview/app_manager/exceptions.py | 2 ++ cartoview/app_manager/installer.py | 5 +---- 3 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 cartoview/app_manager/exceptions.py diff --git a/cartoview/app_manager/decorators.py b/cartoview/app_manager/decorators.py index 43ad3cb1..38bc3d80 100644 --- a/cartoview/app_manager/decorators.py +++ b/cartoview/app_manager/decorators.py @@ -1,7 +1,9 @@ -from .utils import resolve_appinstance from django.conf import settings from django.utils.translation import ugettext as _ +from .exceptions import AppAlreadyInstalledException +from .utils import resolve_appinstance + PERMISSION_MSG_DELETE = _("You are not permitted to delete this Instance") PERMISSION_MSG_GENERIC = _("You do not have permissions for this Instance.") PERMISSION_MSG_MODIFY = _("You are not permitted to modify this Instance") @@ -52,8 +54,9 @@ def wrap(*args, **kwargs): try: return func(*args, **kwargs) except BaseException as e: - if hasattr(this, '_rollback'): - this._rollback() - raise e + if not isinstance(e, AppAlreadyInstalledException) + if hasattr(this, '_rollback'): + this._rollback() + raise e return wrap diff --git a/cartoview/app_manager/exceptions.py b/cartoview/app_manager/exceptions.py new file mode 100644 index 00000000..88479836 --- /dev/null +++ b/cartoview/app_manager/exceptions.py @@ -0,0 +1,2 @@ +class AppAlreadyInstalledException(BaseException): + message = "Application is already installed." diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index 8951a471..47071417 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -26,6 +26,7 @@ from cartoview.store_api.api import StoreAppResource, StoreAppVersion from .decorators import restart_enabled, rollback_on_failure +from .exceptions import AppAlreadyInstalledException from .models import App, AppStore, AppType from .req_installer import ReqInstaller @@ -118,10 +119,6 @@ def get_property_value(self, p): return getattr(self, p, None) -class AppAlreadyInstalledException(BaseException): - message = "Application is already installed." - - class AppInstaller(object): def __init__(self, name, store_id=None, version=None, user=None): self.user = user From ccd5f259d1671d6bef0f1dd18b8437eb8801415f Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 25 Nov 2018 20:54:39 +0200 Subject: [PATCH 22/49] fix invalid syntax error --- cartoview/app_manager/decorators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cartoview/app_manager/decorators.py b/cartoview/app_manager/decorators.py index 38bc3d80..5e0e71ba 100644 --- a/cartoview/app_manager/decorators.py +++ b/cartoview/app_manager/decorators.py @@ -54,9 +54,9 @@ def wrap(*args, **kwargs): try: return func(*args, **kwargs) except BaseException as e: - if not isinstance(e, AppAlreadyInstalledException) - if hasattr(this, '_rollback'): + if not isinstance(e, AppAlreadyInstalledException): + if hasattr(this, '_rollback'): this._rollback() - raise e + raise e return wrap From 031aedb8bab8c8e7d28f86d9b773ba320e5c585e Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 25 Nov 2018 21:03:41 +0200 Subject: [PATCH 23/49] remove redundant else block --- cartoview/app_manager/decorators.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cartoview/app_manager/decorators.py b/cartoview/app_manager/decorators.py index 5e0e71ba..6e233b28 100644 --- a/cartoview/app_manager/decorators.py +++ b/cartoview/app_manager/decorators.py @@ -42,9 +42,6 @@ def restart_enabled(func): def wrap(*args, **kwargs): if not getattr(settings, "CARTOVIEW_TEST", False): return func(*args, **kwargs) - else: - pass - return wrap From 4cd0e90acb20b1bddadee465d789a865847f0f0f Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Mon, 26 Nov 2018 15:34:56 +0200 Subject: [PATCH 24/49] minor improvements - installer.json instead of installer.py - get apps to be migrated from installer.json if available - improve req_installer - move req_installer to apps_handler app --- cartoview/app_manager/installer.py | 17 ++-- cartoview/app_manager/req_installer.py | 36 -------- .../app_manager/tests/test_req_installer.py | 13 ++- cartoview/apps_handler/apps.py | 36 +------- cartoview/apps_handler/apps_operations.py | 85 +++++++++++++++++++ cartoview/apps_handler/req_installer.py | 50 +++++++++++ 6 files changed, 153 insertions(+), 84 deletions(-) delete mode 100644 cartoview/app_manager/req_installer.py create mode 100644 cartoview/apps_handler/apps_operations.py create mode 100644 cartoview/apps_handler/req_installer.py diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index 47071417..a8f0d78d 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -10,7 +10,6 @@ import threading import zipfile from io import BytesIO -from os import R_OK, access from sys import executable, exit from threading import Timer @@ -25,10 +24,12 @@ from cartoview.log_handler import get_logger from cartoview.store_api.api import StoreAppResource, StoreAppVersion +from ..apps_handler.req_installer import (ReqFileException, + ReqFilePermissionException, + ReqInstaller) from .decorators import restart_enabled, rollback_on_failure from .exceptions import AppAlreadyInstalledException from .models import App, AppStore, AppType -from .req_installer import ReqInstaller logger = get_logger(__name__) install_app_batch = getattr(settings, 'INSTALL_APP_BAT', None) @@ -274,12 +275,14 @@ def install(self, restart=True): @rollback_on_failure def _install_requirements(self): - # TODO:add requirement file name as settings var - req_file = os.path.join(self.app_dir, "req.txt") - libs_dir = os.path.join(self.app_dir, "libs") - if os.path.exists(req_file) and access(req_file, R_OK): - req_installer = ReqInstaller(req_file, target=libs_dir) + try: + libs_dir = os.path.join(self.app_dir, 'libs') + req_installer = ReqInstaller(self.app_dir, target=libs_dir) req_installer.install_all() + except BaseException as e: + if not (isinstance(e, ReqFileException) or + isinstance(e, ReqFilePermissionException)): + raise e @rollback_on_failure def check_then_finlize(self, restart, installed_apps): diff --git a/cartoview/app_manager/req_installer.py b/cartoview/app_manager/req_installer.py deleted file mode 100644 index d96895eb..00000000 --- a/cartoview/app_manager/req_installer.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import (absolute_import, division, print_function, - unicode_literals) - -import os -from os import R_OK, access - -from future import standard_library - -standard_library.install_aliases() - - -class ReqFileException(Exception): - message = "requirement file doesn't exists!" - - -class ReqFilePermissionException(Exception): - message = "can not read requirement file!" - - -class ReqInstaller(object): - def __init__(self, reqfile, target=None): - if not os.path.exists(reqfile): - raise ReqFileException() - if not access(reqfile, R_OK): - raise ReqFilePermissionException() - self.file = reqfile - self.requirements = [] - self.target = target - - def install_all(self, *args, **kwargs): - from subprocess import call - command = ["pip", "install", "-r", self.file] - if self.target: - command.extend(["-t", self.target]) - call(command) diff --git a/cartoview/app_manager/tests/test_req_installer.py b/cartoview/app_manager/tests/test_req_installer.py index b63a5b25..22ab4aec 100644 --- a/cartoview/app_manager/tests/test_req_installer.py +++ b/cartoview/app_manager/tests/test_req_installer.py @@ -4,23 +4,22 @@ from django.test import TestCase -from cartoview.app_manager.req_installer import (ReqFileException, - ReqInstaller) +from cartoview.apps_handler.req_installer import ReqFileException, ReqInstaller class ReqInstallerTest(TestCase): def setUp(self): - self.tmp_file = tempfile.NamedTemporaryFile(delete=False) self.tmp_dir = tempfile.mkdtemp() + self.req_file = os.path.join(self.tmp_dir, 'req.txt') + self.target_dir = os.path.join(self.tmp_dir, 'libs') def test_req_installer(self): - with open(self.tmp_file.name, "w") as f: + with open(self.req_file, "w") as f: f.write("pipenv") - req_installer = ReqInstaller(self.tmp_file.name, self.tmp_dir) + req_installer = ReqInstaller(self.tmp_dir, self.target_dir) req_installer.install_all() - self.assertTrue(len(os.listdir(self.tmp_dir)) > 0) + self.assertTrue(len(os.listdir(self.target_dir)) > 0) self.assertRaises(ReqFileException, ReqInstaller, "/dummy_path") def tearDown(self): - os.remove(self.tmp_file.name) shutil.rmtree(self.tmp_dir) diff --git a/cartoview/apps_handler/apps.py b/cartoview/apps_handler/apps.py index 6d5f1b5b..dd988f45 100644 --- a/cartoview/apps_handler/apps.py +++ b/cartoview/apps_handler/apps.py @@ -2,18 +2,13 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import os - from django.apps import AppConfig -from django.conf import settings -from django.core.management import call_command -from django.core.management.base import CommandError from future import standard_library +from cartoview.apps_handler.apps_operations import pending_handler from cartoview.log_handler import get_logger standard_library.install_aliases() -pending_yaml = settings.PENDING_APPS logger = get_logger(__name__) @@ -22,32 +17,5 @@ class AppsHandlerConfig(AppConfig): name = 'cartoview.apps_handler' verbose_name = "Apps Handler" - def delete_application_on_fail(self, appname): - from cartoview.app_manager.installer import AppInstaller - AppInstaller(appname).uninstall(restart=True) - - def execute_pending(self): - from cartoview.apps_handler.handlers import CartoApps, apps_orm - with apps_orm.session() as session: - pending_apps = session.query(CartoApps).filter( - CartoApps.pending == True).all() # noqa - for app in pending_apps: - try: - if not settings.DEBUG: - call_command( - "collectstatic", - interactive=False, - ignore=['node_modules', '.git']) - call_command("migrate", app.name, interactive=False) - except CommandError as e: - error = e.message - logger.error(error) - if error and "does not have migrations" not in error: - self.delete_application_on_fail(app.name) - finally: - CartoApps.set_app_pending(app.name, False) - def ready(self): - apps_dir = getattr(settings, 'APPS_DIR', None) - if apps_dir and os.path.exists(apps_dir): - self.execute_pending() + pending_handler() diff --git a/cartoview/apps_handler/apps_operations.py b/cartoview/apps_handler/apps_operations.py new file mode 100644 index 00000000..7c4b7892 --- /dev/null +++ b/cartoview/apps_handler/apps_operations.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import json +import os + +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import CommandError +from future import standard_library + +from cartoview.log_handler import get_logger + +standard_library.install_aliases() +logger = get_logger(__name__) + + +class AppsHandler(object): + def __init__(self): + pass + + def delete_application_on_fail(self, appname): + from cartoview.app_manager.installer import AppInstaller + AppInstaller(appname).uninstall(restart=True) + + def get_pending_apps(self, app_name): + apps = None + apps_dir = getattr(settings, 'APPS_DIR', None) + _app_dir = os.path.join(apps_dir, app_name) + app_data_file = os.path.join(_app_dir, 'installer.json') + if os.path.exists(app_data_file) and os.access(app_data_file, os.R_OK): + with open(app_data_file, 'r') as f: + app_data = json.load(f) + apps = app_data.get('apps', None) + return apps + + def makemigrations(self, app_name): + call_command("makemigrations", app_name, interactive=False) + + def migrate(self, app_name): + try: + call_command("migrate", app_name, interactive=False) + except CommandError as e: + error = e.message + logger.error(error) + if error and "does not have migrations" not in error: + self.delete_application_on_fail(app_name) + + def collectstatic(self): + if not settings.DEBUG: + call_command( + "collectstatic", + interactive=False, + ignore=['node_modules', '.git']) + + def execute_pending(self): + from cartoview.apps_handler.handlers import CartoApps, apps_orm + with apps_orm.session() as session: + pending_apps = session.query(CartoApps).filter( + CartoApps.pending == True).all() # noqa + for app in pending_apps: + _pending_apps = self.get_pending_apps(app.name) + if _pending_apps: + for _app in _pending_apps: + _app_name = _app.get('name', None) + _make_migrations = _app.get('makemigrations', False) + _migrate = _app.get('makemigrations', False) + if _app_name: + if _make_migrations: + self.makemigrations(_app_name) + if _migrate: + self.migrate(_app_name) + else: + self.migrate(_app_name) + self.collectstatic() + CartoApps.set_app_pending(app.name, False) + + def __call__(self): + apps_dir = getattr(settings, 'APPS_DIR', None) + if apps_dir and os.path.exists(apps_dir): + self.execute_pending() + + +pending_handler = AppsHandler() diff --git a/cartoview/apps_handler/req_installer.py b/cartoview/apps_handler/req_installer.py new file mode 100644 index 00000000..1ec965c0 --- /dev/null +++ b/cartoview/apps_handler/req_installer.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import os +import re + +from future import standard_library + +standard_library.install_aliases() +req_file_regex = re.compile(r'^(req|requirement)s?\.txt') + + +class ReqFileException(Exception): + message = "requirement file doesn't exists!" + + +class ReqFilePermissionException(Exception): + message = "can not read requirement file!" + + +class ReqInstaller(object): + def __init__(self, app_dir, target=None): + self._app_dir = app_dir + self.req_file = None + self.target = target + self.find_req_file() + + def find_req_file(self): + if not os.path.exists(self._app_dir): + raise ReqFileException() + for root, dirs, files in os.walk(self._app_dir): + for name in files: + if req_file_regex.match(name): + self.req_file = os.path.join(root, name) + break + else: + continue + break + if not self.req_file or not os.path.exists(self.req_file): + raise ReqFileException() + if not os.access(self.req_file, os.R_OK): + raise ReqFilePermissionException() + + def install_all(self, *args, **kwargs): + from subprocess import call + command = ["pip", "install", "-r", self.req_file] + if self.target: + command.extend(["-t", self.target]) + call(command) From 0a7cec88847877c75544188da5c46e069c9b10d4 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Wed, 28 Nov 2018 20:47:13 +0200 Subject: [PATCH 25/49] Update Dockerfile to install cartoview from the current branch --- Dockerfile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa2677d8..6cfa10a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,8 @@ ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 RUN apt-get install software-properties-common python-software-properties -y RUN add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable -RUN apt-get update -RUN apt-get upgrade -y -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get upgrade -y +RUN apt-get install -y \ gcc gettext \ python-pip libpq-dev \ sqlite3 git gdal-bin lsof psmisc \ @@ -24,13 +23,19 @@ RUN apt-get update && apt-get install -y \ curl build-essential build-essential python-dev \ --no-install-recommends RUN mkdir /code +COPY . /cartoview +WORKDIR /cartoview +# install cartoview +RUN pip install . +# switch to project dir WORKDIR /code +# remove cartoview +RUN rm -rf /cartoview +# upgrade pip to latest version RUN pip install --upgrade pip +# install additional packages and fix requirements(django-autocomplete-light==2.3.3) RUN pip install --ignore-installed geoip django-geonode-client \ - geonode==2.8rc11 django-jsonfield django-jsonfield-compat \ - cartoview==1.8.3 cherrypy==11.0.0 cheroot==5.8.3 \ django-autocomplete-light==2.3.3 --no-cache-dir -RUN pip install git+https://github.com/GeoNode/django-osgeo-importer.git RUN apt autoremove --purge -y && apt autoclean -y RUN rm -rf ~/.cache/pip RUN rm -rf /var/lib/apt/lists/* && apt-get clean && \ From a5378b3ce9f05249f4241e419d6cac72085c01d4 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Wed, 28 Nov 2018 21:20:59 +0200 Subject: [PATCH 26/49] fix Dockerfile install cartoview --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6cfa10a2..51898b5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,8 @@ RUN apt-get install -y \ python-pylibmc python-setuptools \ curl build-essential build-essential python-dev \ --no-install-recommends +# upgrade pip to latest version +RUN pip install --upgrade pip RUN mkdir /code COPY . /cartoview WORKDIR /cartoview @@ -31,8 +33,6 @@ RUN pip install . WORKDIR /code # remove cartoview RUN rm -rf /cartoview -# upgrade pip to latest version -RUN pip install --upgrade pip # install additional packages and fix requirements(django-autocomplete-light==2.3.3) RUN pip install --ignore-installed geoip django-geonode-client \ django-autocomplete-light==2.3.3 --no-cache-dir From bebdef28f0f87dfe1b7834fe5084337d9cb38638 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sat, 1 Dec 2018 07:19:05 +0200 Subject: [PATCH 27/49] fix create new in app instance page and improve app search --- cartoview/static/cartoview/js/appinstance/search.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cartoview/static/cartoview/js/appinstance/search.js b/cartoview/static/cartoview/js/appinstance/search.js index ce258a77..9237b803 100644 --- a/cartoview/static/cartoview/js/appinstance/search.js +++ b/cartoview/static/cartoview/js/appinstance/search.js @@ -185,7 +185,16 @@ 'type': FILTER_TYPE }; params['single_instance'] = false + if ($location.search().hasOwnProperty('app__name')) { + const oldVal = $location.search()['app__name'] + $location.search('app__name', null) + $location.search('app__name__in', oldVal) + } + if ($location.search().hasOwnProperty('app__title')) { + $location.search('app__title', null) + } if ($location.search().hasOwnProperty('app_name__in')) { + params['app_name__in'] = $location.search()['app__name__in']; } if (compareVersions(angular.version.full, '1.5.0')) { From 09925dbb540395e4fdbd5a9fd2615486da158747 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sat, 1 Dec 2018 07:19:20 +0200 Subject: [PATCH 28/49] fix bulk permissions set --- .../app_manager/templates/app_instance/search_scripts.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cartoview/app_manager/templates/app_instance/search_scripts.html b/cartoview/app_manager/templates/app_instance/search_scripts.html index cb7311b1..c6fa87b2 100644 --- a/cartoview/app_manager/templates/app_instance/search_scripts.html +++ b/cartoview/app_manager/templates/app_instance/search_scripts.html @@ -16,7 +16,10 @@ {% endif %} - +{% if GEONODE_SECURITY_ENABLED %} +{% include "_permissions_form_js.html" %} +{% endif %} + {{block.super}} +