From 1a2ed8adcf2851f53b4881bbc49602f273e021bb Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Thu, 2 Aug 2018 22:31:16 +0200 Subject: [PATCH 01/21] handle markdown on pypi --- README.md | 2 ++ cartoview/__init__.py | 2 +- setup.py | 7 ++----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d325ee1f..58eafdb2 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ ``` - open your browser and type the following address `10.5.0.4` - default user credentials `admin/admin` for cartoview and `admin/geoserver` for geoserver + - you need to configure oauth in geonode and geoserver to do this please use this [link](http://docs.geonode.org/en/master/tutorials/admin/geoserver_geonode_security/index.html) - you can stop containers with `make down` or `docker-compose down` - you can get logs for each service in `docker-compose.yml` unsing the following command: - `docker-compose logs --follow --tail=100 ` @@ -65,5 +66,6 @@ INSTALLED_APPS += load_apps() ``` - restart your server + ## Docs: - [How to use and install](http://cartologic.github.io) diff --git a/cartoview/__init__.py b/cartoview/__init__.py index 3f7699ab..4eb2d954 100644 --- a/cartoview/__init__.py +++ b/cartoview/__init__.py @@ -1,6 +1,6 @@ from .celery import app -__version__ = (1, 8, 2, 'rc', 0) +__version__ = (1, 8, 2, 'rc', 1) __compatible_with__ = [] diff --git a/setup.py b/setup.py index 738d7a6b..0cb7f1ca 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,7 @@ from setuptools import find_packages -try: - import pypandoc - long_description = pypandoc.convert('README.md', 'rst') -except (IOError, ImportError): - long_description = open('README.md').read() +long_description = open('README.md').read() setup( name='cartoview', packages=find_packages(), @@ -14,6 +10,7 @@ description='Cartoview is a GIS web mapping application framework to \ easily share and deploy apps based on Geonode', long_description=long_description, + long_description_content_type='text/markdown', author='Cartologic', author_email='info@cartologic.com', url='https://github.com/cartologic/cartoview', From 110c44f37216aa8917a4af04cd0c91947d7d01fc Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Fri, 3 Aug 2018 01:48:10 +0200 Subject: [PATCH 02/21] use portalocker - Cross-platform locking library --- cartoview/app_manager/installer.py | 28 ++++++++++------------------ cartoview/apps_handler/apps.py | 18 +++++------------- setup.py | 3 ++- 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index cbd62af4..44d0d112 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -3,32 +3,30 @@ unicode_literals) import importlib -import logging import os import shutil import subprocess import tempfile import zipfile -import yaml from builtins import * from io import BytesIO -from .helpers import create_direcotry, change_path_permission -from sys import stdout, exit, executable +from sys import executable, exit from threading import Timer + import pkg_resources +import portalocker import requests +import yaml from django.conf import settings from django.db.models import Max from future import standard_library +from cartoview.log_handler import get_logger + from .config import App as AppConfig -# TODO: find a cross platform function (fcntl is not supported by windows) -try: - import fcntl -except Exception: - pass +from .helpers import change_path_permission, create_direcotry from .models import App, AppStore, AppType -from cartoview.log_handler import get_logger + logger = get_logger(__name__) install_app_batch = getattr(settings, 'INSTALL_APP_BAT', None) standard_library.install_aliases() @@ -43,14 +41,8 @@ def __init__(self): def save_pending_app_to_finlize(self): with open(settings.PENDING_APPS, 'wb') as outfile: - if 'fcntl' in globals(): - fcntl.flock(outfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - yaml.dump(self.apps_to_finlize, outfile, - default_flow_style=False) - fcntl.flock(outfile, fcntl.LOCK_UN) - else: - yaml.dump(self.apps_to_finlize, outfile, - default_flow_style=False) + portalocker.lock(outfile, portalocker.LOCK_EX) + yaml.dump(self.apps_to_finlize, outfile, default_flow_style=False) self.apps_to_finlize = [] def restart_server(self): diff --git a/cartoview/apps_handler/apps.py b/cartoview/apps_handler/apps.py index 64ad1690..b23e87cb 100644 --- a/cartoview/apps_handler/apps.py +++ b/cartoview/apps_handler/apps.py @@ -1,20 +1,16 @@ # -*- coding: utf-8 -*- -import logging import os -from sys import stdout +import portalocker import yaml from django.apps import AppConfig from django.conf import settings from django.core.management import call_command from django.core.management.base import CommandError + from cartoview.log_handler import get_logger -# TODO: find a cross platform function (fcntl is not supported by windows) -try: - import fcntl -except Exception: - pass + pending_yaml = settings.PENDING_APPS logger = get_logger(__name__) @@ -30,12 +26,8 @@ def delete_application_on_fail(self, appname): def reset(self): with open(pending_yaml, 'w+') as f: - if 'fcntl' in globals(): - fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - yaml.dump([], f) - fcntl.flock(f, fcntl.LOCK_UN) - else: - yaml.dump([], f) + portalocker.lock(f, portalocker.LOCK_EX) + yaml.dump([], f) def execute_pending(self): if os.path.exists(pending_yaml): diff --git a/setup.py b/setup.py index 0cb7f1ca..efb99fb3 100644 --- a/setup.py +++ b/setup.py @@ -30,4 +30,5 @@ "Framework :: Django :: 1.8", "Topic :: Scientific/Engineering :: GIS"], license="BSD", - install_requires=['future', 'six==1.10.0', 'geonode>=2.8rc11']) + install_requires=['future', 'six==1.10.0', 'portalocker==1.2.1', + 'geonode>=2.8rc11']) From d9436da68eec3fd2b5f69d9d0fac20a7ba152286 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Fri, 3 Aug 2018 02:09:33 +0200 Subject: [PATCH 03/21] hide app from ordering if not active --- cartoview/app_manager/templates/app_manager/manage.html | 1 + 1 file changed, 1 insertion(+) diff --git a/cartoview/app_manager/templates/app_manager/manage.html b/cartoview/app_manager/templates/app_manager/manage.html index 70266d9c..eed0de6c 100644 --- a/cartoview/app_manager/templates/app_manager/manage.html +++ b/cartoview/app_manager/templates/app_manager/manage.html @@ -165,6 +165,7 @@

  • Date: Fri, 3 Aug 2018 02:33:12 +0200 Subject: [PATCH 04/21] exclude inactive app instance in api --- cartoview/app_manager/models.py | 17 +++++++++-------- cartoview/app_manager/rest.py | 15 +++++++++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cartoview/app_manager/models.py b/cartoview/app_manager/models.py index b14cbc1e..6baa4e00 100644 --- a/cartoview/app_manager/models.py +++ b/cartoview/app_manager/models.py @@ -2,25 +2,26 @@ unicode_literals) import json -import logging from builtins import * -from sys import stdout from datetime import datetime -from django.template.defaultfilters import slugify + from django.conf import settings as geonode_settings from django.contrib.gis.db import models from django.core.urlresolvers import reverse from django.db.models import signals +from django.template.defaultfilters import slugify from future import standard_library -from jsonfield import JSONField -from taggit.managers import TaggableManager # Create your models here. from geonode.base.models import ResourceBase, resourcebase_post_save from geonode.maps.models import Map as GeonodeMap -from geonode.security.models import (remove_object_permissions) +from geonode.security.models import remove_object_permissions +from jsonfield import JSONField +from taggit.managers import TaggableManager -from .config import AppsConfig from cartoview.log_handler import get_logger + +from .config import AppsConfig + logger = get_logger(__name__) standard_library.install_aliases() @@ -161,7 +162,7 @@ def name_long(self): return str(self.id) else: return '%s (%s)' % (self.title, self.id) - + @property def config_obj(self): try: diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index a3368f0d..9dd02e3e 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -3,15 +3,17 @@ unicode_literals) import json -from cartoview.app_manager.models import App, AppInstance, AppStore, AppType + from django.conf import settings from django.conf.urls import url from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse +from django.db.models import Q from future import standard_library from geonode.api.api import ProfileResource from geonode.api.authorization import GeoNodeAuthorization -from geonode.api.resourcebase_api import CommonMetaApi +from geonode.api.resourcebase_api import (CommonMetaApi, LayerResource, + MapResource) from geonode.layers.models import Attribute from geonode.maps.models import MapLayer from geonode.people.models import Profile @@ -23,8 +25,10 @@ from tastypie.resources import ModelResource from tastypie.utils import trailing_slash +from cartoview.app_manager.models import App, AppInstance, AppStore, AppType + from .resources import FileUploadResource -from geonode.api.resourcebase_api import LayerResource, MapResource + standard_library.install_aliases() @@ -196,10 +200,13 @@ class AppInstanceResource(ModelResource): keywords = fields.ListField(null=True, blank=True) class Meta(CommonMetaApi): + __inactive_apps = [ + app.id for app in App.objects.all() if not app.config.active] filtering = CommonMetaApi.filtering always_return_data = True filtering.update({'app': ALL_WITH_RELATIONS, 'featured': ALL}) - queryset = AppInstance.objects.distinct().order_by('-date') + queryset = AppInstance.objects.distinct().exclude( + Q(app__isnull=True) | Q(app__id__in=__inactive_apps)).order_by('-date') if settings.RESOURCE_PUBLISHING: queryset = queryset.filter(is_published=True) resource_name = 'appinstances' From 25d73ce714506cdc2d8172b544401b3590eabf7b Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Fri, 3 Aug 2018 02:48:12 +0200 Subject: [PATCH 05/21] exclude inactive app_instance from ResourceBaseResource --- cartoview/cartoview_api/rest.py | 36 +++++++++++++++++++++++---------- cartoview/urls.py | 15 ++++++++------ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cartoview/cartoview_api/rest.py b/cartoview/cartoview_api/rest.py index 8095801e..e3002159 100644 --- a/cartoview/cartoview_api/rest.py +++ b/cartoview/cartoview_api/rest.py @@ -1,18 +1,19 @@ -from django.core.urlresolvers import reverse -from tastypie import fields -from tastypie.constants import ALL, ALL_WITH_RELATIONS -from tastypie.resources import ModelResource import json + +from django.core.urlresolvers import reverse from geonode.api.api import OwnersResource -from geonode.api.resourcebase_api import MapResource +from geonode.api.authorization import (GeonodeApiKeyAuthentication, + GeoNodeAuthorization) +from geonode.api.resourcebase_api import MapResource, ResourceBaseResource from geonode.base.models import ResourceBase -from cartoview.app_manager.rest import LayerFilterExtensionResource +from geonode.layers.models import Attribute, Layer from geonode.maps.models import MapLayer -from geonode.layers.models import Attribute +from tastypie import fields from tastypie.authentication import MultiAuthentication, SessionAuthentication -from geonode.api.authorization import ( - GeoNodeAuthorization, GeonodeApiKeyAuthentication) -from geonode.layers.models import Layer +from tastypie.constants import ALL, ALL_WITH_RELATIONS +from tastypie.resources import ModelResource +from cartoview.app_manager.models import App, AppInstance +from cartoview.app_manager.rest import LayerFilterExtensionResource type_filter = { 'app': 'appinstance', @@ -22,6 +23,19 @@ } +class ExtendedResourceBaseResource(ResourceBaseResource): + class Meta(ResourceBaseResource.Meta): + __inactive_apps = [ + app.id for app in App.objects.all() if not app.config.active] + __inactive_apps_instances = [instance.id for instance in + AppInstance.objects.filter( + app__id__in=__inactive_apps)] + queryset = ResourceBase.objects.polymorphic_queryset() \ + .distinct().exclude( + id__in=__inactive_apps_instances).order_by('-date') + resource_name = 'base' + + class AllResourcesResource(ModelResource): type = fields.CharField(null=False, blank=False) app = fields.DictField(null=True, blank=False) @@ -33,7 +47,7 @@ class Meta: resource_name = 'all_resources' queryset = ResourceBase.objects.distinct() fields = ['id', 'title', 'abstract', - 'type', 'featured', 'owner__username','app', 'owner', 'urls', 'thumbnail_url'] + 'type', 'featured', 'owner__username', 'app', 'owner', 'urls', 'thumbnail_url'] filtering = { 'id': ALL, 'title': ALL, diff --git a/cartoview/urls.py b/cartoview/urls.py index c03668a1..8042c1e2 100644 --- a/cartoview/urls.py +++ b/cartoview/urls.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- -from django.conf.urls import patterns, url, include +from django.conf.urls import include, patterns, url +from geonode.api.urls import api from geonode.urls import urlpatterns -from cartoview.cartoview_api.views import layer_config_json, update_extent -from cartoview_api.rest import ( - AllResourcesResource, AttributeResource, MapLayerResource) + from cartoview.app_manager.rest import (AppInstanceResource, AppResource, AppTypeResource, LayerFilterExtensionResource) -from cartoview.views import index as cartoview_index, check_version -from geonode.api.urls import api +from cartoview.cartoview_api.views import layer_config_json, update_extent +from cartoview.views import check_version +from cartoview.views import index as cartoview_index +from cartoview_api.rest import (AllResourcesResource, AttributeResource, + ExtendedResourceBaseResource, MapLayerResource) api.register(AppInstanceResource()) api.register(AppResource()) @@ -17,6 +19,7 @@ api.register(AllResourcesResource()) api.register(AttributeResource()) api.register(MapLayerResource()) +api.register(ExtendedResourceBaseResource()) urlpatterns = patterns( '', url(r'^/?$', cartoview_index, name='home'), From 7b4b1e2cdd42ae88279ac0022d8dc037d45a85c0 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Fri, 3 Aug 2018 03:06:41 +0200 Subject: [PATCH 06/21] fix inactive app_instance exclude from AppInstanceResource/ResourceBase --- cartoview/app_manager/rest.py | 18 ++++++++++++++---- cartoview/cartoview_api/rest.py | 18 +++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index 9dd02e3e..de4805d9 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -200,13 +200,10 @@ class AppInstanceResource(ModelResource): keywords = fields.ListField(null=True, blank=True) class Meta(CommonMetaApi): - __inactive_apps = [ - app.id for app in App.objects.all() if not app.config.active] filtering = CommonMetaApi.filtering always_return_data = True filtering.update({'app': ALL_WITH_RELATIONS, 'featured': ALL}) - queryset = AppInstance.objects.distinct().exclude( - Q(app__isnull=True) | Q(app__id__in=__inactive_apps)).order_by('-date') + queryset = AppInstance.objects.distinct().order_by('-date') if settings.RESOURCE_PUBLISHING: queryset = queryset.filter(is_published=True) resource_name = 'appinstances' @@ -214,6 +211,19 @@ class Meta(CommonMetaApi): excludes = ['csw_anytext', 'metadata_xml'] authorization = GeoNodeAuthorization() + def get_object_list(self, request): + __inactive_apps = [ + app.id for app in App.objects.all() if not app.config.active] + __inactive_apps_instances = [instance.id for instance in + AppInstance.objects.filter( + app__id__in=__inactive_apps)] + active_app_instances = super(AppInstanceResource, self)\ + .get_object_list( + request).exclude( + id__in=__inactive_apps_instances) + + return active_app_instances + def dehydrate_owner(self, bundle): return bundle.obj.owner.username diff --git a/cartoview/cartoview_api/rest.py b/cartoview/cartoview_api/rest.py index e3002159..32010113 100644 --- a/cartoview/cartoview_api/rest.py +++ b/cartoview/cartoview_api/rest.py @@ -30,11 +30,23 @@ class Meta(ResourceBaseResource.Meta): __inactive_apps_instances = [instance.id for instance in AppInstance.objects.filter( app__id__in=__inactive_apps)] - queryset = ResourceBase.objects.polymorphic_queryset() \ - .distinct().exclude( - id__in=__inactive_apps_instances).order_by('-date') + queryset = ResourceBase.objects.polymorphic_queryset().distinct()\ + .order_by('-date') resource_name = 'base' + def get_object_list(self, request): + __inactive_apps = [ + app.id for app in App.objects.all() if not app.config.active] + __inactive_apps_instances = [instance.id for instance in + AppInstance.objects.filter( + app__id__in=__inactive_apps)] + active_app_instances = super(ExtendedResourceBaseResource, self)\ + .get_object_list( + request).exclude( + id__in=__inactive_apps_instances) + + return active_app_instances + class AllResourcesResource(ModelResource): type = fields.CharField(null=False, blank=False) From 739ed03e06f2173a450c252214956421b86b6e39 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Fri, 3 Aug 2018 03:14:21 +0200 Subject: [PATCH 07/21] populate apps on active change --- cartoview/app_manager/rest.py | 3 ++- cartoview/app_manager/utils.py | 14 ++++++++++---- cartoview/cartoview_api/rest.py | 5 ----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index de4805d9..41abe8fe 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -8,7 +8,6 @@ from django.conf.urls import url from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse -from django.db.models import Q from future import standard_library from geonode.api.api import ProfileResource from geonode.api.authorization import GeoNodeAuthorization @@ -28,6 +27,7 @@ from cartoview.app_manager.models import App, AppInstance, AppStore, AppType from .resources import FileUploadResource +from .utils import populate_apps standard_library.install_aliases() @@ -153,6 +153,7 @@ def set_active(self, active, request, **kwargs): bundle=bundle, **self.remove_api_resource_names(kwargs)) app.config.active = active app.apps_config.save() + populate_apps() except ObjectDoesNotExist: return HttpGone() diff --git a/cartoview/app_manager/utils.py b/cartoview/app_manager/utils.py index ca02ad81..12b40467 100644 --- a/cartoview/app_manager/utils.py +++ b/cartoview/app_manager/utils.py @@ -7,16 +7,16 @@ from django.http import HttpResponse from django.shortcuts import get_object_or_404 +from django.utils.translation import ugettext as _ from django.views.decorators.http import require_http_methods from future import standard_library from future.utils import with_metaclass -from tastypie.serializers import Serializer - -from cartoview.app_manager.models import AppInstance from geonode.api.resourcebase_api import LayerResource from geonode.maps.models import Map -from django.utils.translation import ugettext as _ from geonode.utils import resolve_object +from tastypie.serializers import Serializer + +from cartoview.app_manager.models import AppInstance standard_library.install_aliases() @@ -105,3 +105,9 @@ def create_thumbnail(self): parent_app_thumbnail_url = instance.map.get_thumbnail_url() instance.thumbnail_url = parent_app_thumbnail_url instance.save() + + +def populate_apps(): + from django.apps import apps + from django.conf import settings + apps.populate(settings.INSTALLED_APPS) diff --git a/cartoview/cartoview_api/rest.py b/cartoview/cartoview_api/rest.py index 32010113..2d94fd34 100644 --- a/cartoview/cartoview_api/rest.py +++ b/cartoview/cartoview_api/rest.py @@ -25,11 +25,6 @@ class ExtendedResourceBaseResource(ResourceBaseResource): class Meta(ResourceBaseResource.Meta): - __inactive_apps = [ - app.id for app in App.objects.all() if not app.config.active] - __inactive_apps_instances = [instance.id for instance in - AppInstance.objects.filter( - app__id__in=__inactive_apps)] queryset = ResourceBase.objects.polymorphic_queryset().distinct()\ .order_by('-date') resource_name = 'base' From ffce75129304b066a75eda66540c613bd2f33078 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sat, 4 Aug 2018 19:47:33 +0200 Subject: [PATCH 08/21] add apps filter in app instance page --- .../app_instance/_search_content.html | 16 +++ .../app_manager/app_instance_list.html | 7 +- .../static/cartoview/js/appinstance/search.js | 100 +++++++++++------- 3 files changed, 84 insertions(+), 39 deletions(-) diff --git a/cartoview/app_manager/templates/app_instance/_search_content.html b/cartoview/app_manager/templates/app_instance/_search_content.html index 0c742e67..a07c2ba0 100644 --- a/cartoview/app_manager/templates/app_instance/_search_content.html +++ b/cartoview/app_manager/templates/app_instance/_search_content.html @@ -25,6 +25,22 @@
    {% include "search/_general_filters.html" %}
    +
    + +
    diff --git a/cartoview/app_manager/templates/app_manager/app_instance_list.html b/cartoview/app_manager/templates/app_manager/app_instance_list.html index 47d2f56c..3775f8ff 100644 --- a/cartoview/app_manager/templates/app_manager/app_instance_list.html +++ b/cartoview/app_manager/templates/app_manager/app_instance_list.html @@ -4,7 +4,10 @@ {% load url from future %} {% block title %} {% trans "Explore Apps" %} - {{ block.super }} {% endblock %} - +{% block head %} + {{ block.super }} + +{% endblock %} {% block body_class %}appinstances appinstances-list explore{% endblock %} {% block body %} @@ -47,8 +50,10 @@

    {% verbatim %}{{ appTitle }}{% endverbatim %}

    diff --git a/cartoview/static/cartoview/js/appinstance/search.js b/cartoview/static/cartoview/js/appinstance/search.js index 4f2ca64e..5ed9a312 100644 --- a/cartoview/static/cartoview/js/appinstance/search.js +++ b/cartoview/static/cartoview/js/appinstance/search.js @@ -3,9 +3,9 @@ */ 'use strict'; -(function() { +(function () { - var module = angular.module('geonode_main_search', [], function( + var module = angular.module('geonode_main_search', [], function ( $locationProvider) { if (window.navigator.userAgent.indexOf("MSIE") == -1) { $locationProvider.html5Mode({ @@ -19,7 +19,7 @@ }); // Used to set the class of the filters based on the url parameters - module.set_initial_filters_from_query = function(data, url_query, + module.set_initial_filters_from_query = function (data, url_query, filter_param) { for (var i = 0; i < data.length; i++) { if (url_query == data[i][filter_param] || url_query.indexOf(data[i][ @@ -34,7 +34,7 @@ } // Load categories, keywords, and regions - module.load_categories = function($http, $rootScope, $location) { + module.load_categories = function ($http, $rootScope, $location) { var params = typeof FILTER_TYPE == 'undefined' ? {} : { 'type': FILTER_TYPE }; @@ -43,7 +43,7 @@ } $http.get(CATEGORIES_ENDPOINT, { params: params - }).success(function(data) { + }).success(function (data) { if ($location.search().hasOwnProperty('category__identifier__in')) { data.objects = module.set_initial_filters_from_query(data.objects, $location.search()['category__identifier__in'], 'identifier'); @@ -55,7 +55,7 @@ }); } - module.load_keywords = function($http, $rootScope, $location) { + module.load_keywords = function ($http, $rootScope, $location) { var params = typeof FILTER_TYPE == 'undefined' ? {} : { 'type': FILTER_TYPE }; @@ -64,7 +64,7 @@ } $http.get(KEYWORDS_ENDPOINT, { params: params - }).success(function(data) { + }).success(function (data) { if ($location.search().hasOwnProperty('keywords__slug__in')) { data.objects = module.set_initial_filters_from_query(data.objects, $location.search()['keywords__slug__in'], 'slug'); @@ -76,7 +76,7 @@ }); } - module.load_regions = function($http, $rootScope, $location) { + module.load_regions = function ($http, $rootScope, $location) { var params = typeof FILTER_TYPE == 'undefined' ? {} : { 'type': FILTER_TYPE }; @@ -85,7 +85,7 @@ } $http.get(REGIONS_ENDPOINT, { params: params - }).success(function(data) { + }).success(function (data) { if ($location.search().hasOwnProperty('regions__name__in')) { data.objects = module.set_initial_filters_from_query(data.objects, $location.search()['regions__name__in'], 'name'); @@ -97,7 +97,7 @@ }); } - module.load_owners = function($http, $rootScope, $location) { + module.load_owners = function ($http, $rootScope, $location) { var params = typeof FILTER_TYPE == 'undefined' ? {} : { 'type': FILTER_TYPE }; @@ -106,7 +106,7 @@ } $http.get(OWNERS_ENDPOINT, { params: params - }).success(function(data) { + }).success(function (data) { if ($location.search().hasOwnProperty('owner__username__in')) { data.objects = module.set_initial_filters_from_query(data.objects, $location.search()['owner__username__in'], 'identifier'); @@ -117,9 +117,30 @@ } }); } + module.load_apps = function ($http, $rootScope, $location) { + var params = typeof FILTER_TYPE == 'undefined' ? {} : { + 'type': FILTER_TYPE + }; + params['single_instance'] = false + if ($location.search().hasOwnProperty('app_name__in')) { + params['app_name__in'] = $location.search()['app__name__in']; + } + $http.get(APPS_ENDPOINT, { + params: params + }).success(function (data) { + if ($location.search().hasOwnProperty('app__name__in')) { + data.objects = module.set_initial_filters_from_query(data.objects, + $location.search()['app__name__in'], 'identifier'); + } + $rootScope.apps = data.objects; + if (HAYSTACK_FACET_COUNTS && $rootScope.query_data) { + module.haystack_facets($http, $rootScope, $location); + } + }); + } // Update facet counts for categories and keywords - module.haystack_facets = function($http, $rootScope, $location) { + module.haystack_facets = function ($http, $rootScope, $location) { var data = $rootScope.query_data; if ("categories" in $rootScope) { $rootScope.category_counts = data.meta.facets.category; @@ -173,7 +194,7 @@ /* * Load categories and keywords */ - module.run(function($http, $rootScope, $location) { + module.run(function ($http, $rootScope, $location) { /* * Load categories and keywords if the filter is available in the page * and set active class if needed @@ -190,6 +211,9 @@ if ($('#owners').length > 0) { module.load_owners($http, $rootScope, $location); } + if ($('#apps').length > 0) { + module.load_apps($http, $rootScope, $location); + } // Activate the type filters if in the url @@ -221,9 +245,9 @@ * Load data from api and defines the multiple and single choice handlers * Syncs the browser url with the selections */ - module.controller('geonode_search_controller', function($injector, $scope, + module.controller('geonode_search_controller', function ($injector, $scope, $location, $http, Configs, $parse) { - $scope.loading=true + $scope.loading = true $scope.query = $location.search(); $scope.query.limit = $scope.query.limit || CLIENT_RESULTS_LIMIT; $scope.query.offset = $scope.query.offset || 0; @@ -237,7 +261,7 @@ params: { name: data.app__name } - }).success(function(res) { + }).success(function (res) { $scope.app_name = res.objects[0].name; }) } else { @@ -245,9 +269,9 @@ } $http.get(Configs.url, { params: data || {} - }).success(function(data) { - $scope.loading=false - $scope.results = data.objects.sort(function(x, y) { + }).success(function (data) { + $scope.loading = false + $scope.results = data.objects.sort(function (x, y) { return (x.featured === y.featured) ? 0 : x.featured ? -1 : 1; }); $scope.total_counts = data.meta.total_count; @@ -266,7 +290,7 @@ //Update facet/keyword/category counts from search results if (HAYSTACK_FACET_COUNTS) { module.haystack_facets($http, $scope.$root, $location); - $("#types").find("a").each(function() { + $("#types").find("a").each(function () { if ($(this)[0].id in data.meta.facets.subtype) { $(this).find("span").text(data.meta.facets.subtype[$(this)[0].id]); } else if ($(this)[0].id in data.meta.facets.type) { @@ -285,7 +309,7 @@ * Pagination */ // Control what happens when the total results change - $scope.$watch('total_counts', function() { + $scope.$watch('total_counts', function () { $scope.numpages = Math.round( ($scope.total_counts / $scope.query.limit) + 0.49 ); @@ -305,7 +329,7 @@ }; }); - $scope.paginate_down = function() { + $scope.paginate_down = function () { if ($scope.page > 1) { $scope.page -= 1; $scope.query.offset = $scope.query.limit * ($scope.page - 1); @@ -313,7 +337,7 @@ } } - $scope.paginate_up = function() { + $scope.paginate_up = function () { if ($scope.numpages > $scope.page) { $scope.page += 1; $scope.query.offset = $scope.query.limit * ($scope.page - 1); @@ -327,7 +351,7 @@ if (!Configs.hasOwnProperty("disableQuerySync")) { // Keep in sync the page location with the query object - $scope.$watch('query', function() { + $scope.$watch('query', function () { $location.search($scope.query); }, true); } @@ -336,7 +360,7 @@ * Add the selection behavior to the element, it adds/removes the 'active' class * and pushes/removes the value of the element from the query object */ - $scope.multiple_choice_listener = function($event) { + $scope.multiple_choice_listener = function ($event) { var element = $($event.target); var query_entry = []; var data_filter = element.attr('data-filter'); @@ -382,7 +406,7 @@ query_api($scope.query); } - $scope.single_choice_listener = function($event) { + $scope.single_choice_listener = function ($event) { var element = $($event.target); var query_entry = []; var data_filter = element.attr('data-filter'); @@ -421,14 +445,14 @@ autoHilightFirst: false }); - $('#text_search_input').keypress(function(e) { + $('#text_search_input').keypress(function (e) { if (e.which == 13) { $('#text_search_btn').click(); $('.yourlabs-autocomplete').hide(); } }); - $('#text_search_input').bind('selectChoice', function(e, choice, + $('#text_search_input').bind('selectChoice', function (e, choice, text_autocomplete) { if (choice[0].children[0] == undefined) { $('#text_search_input').val($(choice[0]).text()); @@ -436,7 +460,7 @@ } }); - $('#text_search_btn').click(function() { + $('#text_search_btn').click(function () { if (HAYSTACK_SEARCH) $scope.query['q'] = $('#text_search_input').val(); else @@ -455,7 +479,7 @@ appendAutocomplete: $('#region_search_input'), placeholder: gettext('Enter your region here ...') }); - $('#region_search_input').bind('selectChoice', function(e, choice, + $('#region_search_input').bind('selectChoice', function (e, choice, region_autocomplete) { if (choice[0].children[0] == undefined) { $('#region_search_input').val(choice[0].innerHTML); @@ -463,12 +487,12 @@ } }); - $('#region_search_btn').click(function() { + $('#region_search_btn').click(function () { $scope.query['regions__name__in'] = $('#region_search_input').val(); query_api($scope.query); }); - $scope.feature_select = function($event) { + $scope.feature_select = function ($event) { var element = $($event.target); var article = $(element.parents('article')[0]); if (article.hasClass('resource_selected')) { @@ -489,7 +513,7 @@ 'date__lte': '' }; var init_date = true; - $scope.$watch('date_query', function() { + $scope.$watch('date_query', function () { if ($scope.date_query.date__gte != '' && $scope.date_query.date__lte != '') { $scope.query['date__range'] = $scope.date_query.date__gte + ',' + @@ -550,22 +574,22 @@ var leafletData = $injector.get('leafletData'), map = leafletData.getMap('filter-map'); - map.then(function(map) { - map.on('moveend', function() { + map.then(function (map) { + map.on('moveend', function () { $scope.query['extent'] = map.getBounds().toBBoxString(); query_api($scope.query); }); }); var showMap = false; - $('#_extent_filter').click(function(evt) { + $('#_extent_filter').click(function (evt) { showMap = !showMap if (showMap) { - leafletData.getMap().then(function(map) { + leafletData.getMap().then(function (map) { map.invalidateSize(); }); } }); } }); -})(); +})(); \ No newline at end of file From 7693248967245fec8abd7f475d6c75d84c180753 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sat, 4 Aug 2018 20:55:39 +0200 Subject: [PATCH 09/21] better error handling --- cartoview/app_manager/helpers.py | 4 ++++ cartoview/app_manager/installer.py | 12 ++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cartoview/app_manager/helpers.py b/cartoview/app_manager/helpers.py index 6a6f376a..71c8a875 100644 --- a/cartoview/app_manager/helpers.py +++ b/cartoview/app_manager/helpers.py @@ -1,6 +1,8 @@ import os import stat world_permission = 0o777 +from cartoview.log_handler import get_logger +logger = get_logger(__name__) def create_direcotry(path, mode=0777): @@ -10,6 +12,8 @@ def create_direcotry(path, mode=0777): try: previous_mask = os.umask(0) os.makedirs(path, mode=mode) + except OSError as e: + logger.error(e.message) finally: # set the previous mask back os.umask(previous_mask) diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index 44d0d112..d1df0331 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -149,13 +149,9 @@ def _request_rest_data(self, *args): """ get app information form app store rest url """ - try: - q = requests.get(self.store.url + ''.join( - [str(item) for item in args])) - return q.json() - except Exception as e: - logger.error(e.message) - return None + q = requests.get(self.store.url + ''.join( + [str(item) for item in args])) + return q.json() def extract_move_app(self, zipped_app): extract_to = tempfile.mkdtemp(dir=temp_dir) @@ -180,7 +176,7 @@ def _download_app(self): if not os.access(temp_dir, os.W_OK): change_path_permission(temp_dir) self.extract_move_app(zip_ref) - except Exception as e: + except shutil.Error as e: raise e finally: zip_ref.close() From d6d3ee5808dc578159fe737230a3943846aa1901 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sat, 4 Aug 2018 20:56:02 +0200 Subject: [PATCH 10/21] return valid js version --- cartoview/app_manager/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cartoview/app_manager/views.py b/cartoview/app_manager/views.py index 5a114a70..b310cf1e 100644 --- a/cartoview/app_manager/views.py +++ b/cartoview/app_manager/views.py @@ -101,12 +101,16 @@ def installed_apps(): @staff_member_required def manage_apps(request): from cartoview.version import get_backward_compatible, get_current_version + from pkg_resources import parse_version apps = App.objects.all() + _version = parse_version(get_current_version())._version + release = _version.release + version = [str(x) for x in release] context = { 'apps': apps, 'site_apps': get_apps_names(), 'version_info': { - 'current_version': get_current_version(), + 'current_version': ".".join(version), 'backward_versions': get_backward_compatible() } } From bd3d9bdce2bfee11797885f7d37552f447cebc9f Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sat, 4 Aug 2018 21:36:43 +0200 Subject: [PATCH 11/21] Deprecated since version 1.8: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support for string view arguments is deprecated and will be removed in Django 1.10. Pass the callable instead. The prefix parameter has the same meaning as the first argument to patterns() and is only relevant when you’re passing a string as the view parameter. patterns is deprecated since 1.10 --- cartoview/app_manager/api.py | 9 +++++---- cartoview/app_manager/urls.py | 9 ++++----- cartoview/app_manager/views.py | 20 ++++++++++---------- cartoview/apps_handler/apps.py | 2 -- cartoview/cartoview_api/views.py | 14 ++++++++------ cartoview/celery.py | 2 +- cartoview/log_handler.py | 3 ++- cartoview/urls.py | 9 ++++----- cartoview/version.py | 4 +++- cartoview/views.py | 4 +++- 10 files changed, 40 insertions(+), 36 deletions(-) diff --git a/cartoview/app_manager/api.py b/cartoview/app_manager/api.py index 75c5637e..9abf3783 100644 --- a/cartoview/app_manager/api.py +++ b/cartoview/app_manager/api.py @@ -3,7 +3,7 @@ import warnings from builtins import * -from django.conf.urls import include, patterns, url +from django.conf.urls import include, url from django.shortcuts import render from future import standard_library from tastypie.api import Api as TastypieApi @@ -47,7 +47,8 @@ def urls(self): for name in sorted(self._registry.keys()): resource = self._registry[name] resource.api_name = self.api_name - pattern_list.append((r"^%s" % api_pattern, include(resource.urls))) + pattern_list.append( + url(r"^%s" % api_pattern, include(resource.urls))) urlpatterns = self.prepend_urls() overridden_urls = self.override_urls() @@ -60,7 +61,7 @@ def urls(self): urlpatterns += overridden_urls - urlpatterns += patterns('', *pattern_list) + urlpatterns += pattern_list return urlpatterns @@ -91,7 +92,7 @@ def urls(self): for name in sorted(self.apis.keys()): pattern_list.append( url(r"^%s/" % name, include(self.apis[name].urls))) - self.urlpatterns = patterns('', *pattern_list) + self.urlpatterns = pattern_list return self.urlpatterns def register_app_urls(self, app_name): diff --git a/cartoview/app_manager/urls.py b/cartoview/app_manager/urls.py index a624a723..cc83b677 100644 --- a/cartoview/app_manager/urls.py +++ b/cartoview/app_manager/urls.py @@ -5,7 +5,7 @@ import importlib from builtins import * -from django.conf.urls import include, patterns, url +from django.conf.urls import include, url from django.views.generic import TemplateView from future import standard_library @@ -37,8 +37,7 @@ # # TODO: log the error # pass -urlpatterns = patterns( - 'cartoview.app_manager', +urlpatterns = [ url(r'^$', app_manager_views.index, name='app_manager_base_url'), url(r'^manage/$', app_manager_views.manage_apps, name='manage_apps'), url(r'^install/(?P\d+)/(?P.*)/(?P.*)/$', @@ -67,8 +66,8 @@ url(r'^(?P\d+)/remove$', app_manager_views.appinstance_remove, name="appinstance_remove"), - (r'^rest/', include(rest_api.urls)), - url(r'^maplayers/api$', map_layers, name="cartoview_map_layers"), ) + url(r'^rest/', include(rest_api.urls)), + url(r'^maplayers/api$', map_layers, name="cartoview_map_layers"), ] def import_app_rest(app_name): diff --git a/cartoview/app_manager/views.py b/cartoview/app_manager/views.py index b310cf1e..1f124eb6 100644 --- a/cartoview/app_manager/views.py +++ b/cartoview/app_manager/views.py @@ -10,7 +10,7 @@ from urllib.parse import urljoin from django.conf import settings -from django.conf.urls import patterns, url +from django.conf.urls import url from django.contrib.admin.views.decorators import staff_member_required from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied @@ -617,12 +617,12 @@ def view_app(self, request, instance_id, template=None, context={}): return render(request, template, context) def get_url_patterns(self): - return patterns('', - url(r'^new/$', self.new, - name='%s.new' % self.app_name), - url(r'^(?P\d+)/edit/$', - self.edit, name='%s.edit' % self.app_name), - url(r'^(?P\d+)/view/$', - self.view_app, - name='%s.view' % self.app_name) - ) + return [ + url(r'^new/$', self.new, + name='%s.new' % self.app_name), + url(r'^(?P\d+)/edit/$', + self.edit, name='%s.edit' % self.app_name), + url(r'^(?P\d+)/view/$', + self.view_app, + name='%s.view' % self.app_name) + ] diff --git a/cartoview/apps_handler/apps.py b/cartoview/apps_handler/apps.py index b23e87cb..a08ee1c3 100644 --- a/cartoview/apps_handler/apps.py +++ b/cartoview/apps_handler/apps.py @@ -46,8 +46,6 @@ def execute_pending(self): not in error: self.delete_application_on_fail(app) self.reset() - else: - pass def ready(self): self.execute_pending() diff --git a/cartoview/cartoview_api/views.py b/cartoview/cartoview_api/views.py index 73bd43e1..5aa36bf1 100644 --- a/cartoview/cartoview_api/views.py +++ b/cartoview/cartoview_api/views.py @@ -1,14 +1,16 @@ +import requests from django.shortcuts import HttpResponse -from geonode.layers.views import layer_detail +from geonode.geoserver.helpers import ogc_server_settings +from geonode.layers.views import (_PERMISSION_MSG_MODIFY, _resolve_layer, + layer_detail) +from requests.auth import HTTPBasicAuth + +from cartoview.log_handler import get_logger + try: import simplejson as json except: import json -import requests -from geonode.geoserver.helpers import ogc_server_settings -from requests.auth import HTTPBasicAuth -from geonode.layers.views import _resolve_layer, _PERMISSION_MSG_MODIFY -from cartoview.log_handler import get_logger logger = get_logger(__name__) diff --git a/cartoview/celery.py b/cartoview/celery.py index 0d78ac86..6fa1a0c0 100644 --- a/cartoview/celery.py +++ b/cartoview/celery.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import os + from celery import Celery -from django.conf import settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cartoview.settings') diff --git a/cartoview/log_handler.py b/cartoview/log_handler.py index 8408f3a1..2d119728 100644 --- a/cartoview/log_handler.py +++ b/cartoview/log_handler.py @@ -1,5 +1,6 @@ -from sys import stdout import logging +from sys import stdout + formatter = logging.Formatter( '[%(asctime)s] p%(process)s { %(name)s %(pathname)s:%(lineno)d} \ %(levelname)s - %(message)s', '%m-%d %H:%M:%S') diff --git a/cartoview/urls.py b/cartoview/urls.py index 8042c1e2..5167acbf 100644 --- a/cartoview/urls.py +++ b/cartoview/urls.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from django.conf.urls import include, patterns, url +from django.conf.urls import include, url from geonode.api.urls import api -from geonode.urls import urlpatterns +from geonode.urls import urlpatterns as geonode_urls from cartoview.app_manager.rest import (AppInstanceResource, AppResource, AppTypeResource, @@ -20,8 +20,7 @@ api.register(AttributeResource()) api.register(MapLayerResource()) api.register(ExtendedResourceBaseResource()) -urlpatterns = patterns( - '', +urlpatterns = [ url(r'^/?$', cartoview_index, name='home'), url(r'^layer/(?P[^/]*)/json/?$', layer_config_json, name='layer_json'), @@ -29,4 +28,4 @@ name='cartoview.update_extent'), url(r'^check-version/$', check_version, name='check_version'), url(r'', include(api.urls)), - (r'^apps/', include('cartoview.app_manager.urls')),) + urlpatterns + url(r'^apps/', include('cartoview.app_manager.urls')), ] + geonode_urls diff --git a/cartoview/version.py b/cartoview/version.py index 365deaab..16a807a5 100644 --- a/cartoview/version.py +++ b/cartoview/version.py @@ -1,6 +1,8 @@ +import json + from geonode.version import get_version + from cartoview import __compatible_with__, __version__ -import json def get_current_version(): diff --git a/cartoview/views.py b/cartoview/views.py index 13b9de50..a82c1164 100644 --- a/cartoview/views.py +++ b/cartoview/views.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- import requests from django.shortcuts import render -from .version import (get_current_version) + from .log_handler import get_logger +from .version import get_current_version + logger = get_logger(__name__) From 18f18d7d4ab22f3f802297d6cf8e84b5319288e8 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 00:20:58 +0200 Subject: [PATCH 12/21] better handling for app installation --- cartoview/app_manager/installer.py | 35 +++++++++++++++---- .../static/app_manager/js/manager.js | 3 ++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index d1df0331..0d35bd3b 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -18,6 +18,7 @@ import requests import yaml from django.conf import settings +from django.db import transaction from django.db.models import Max from future import standard_library @@ -177,6 +178,7 @@ def _download_app(self): change_path_permission(temp_dir) self.extract_move_app(zip_ref) except shutil.Error as e: + logger.error(e.message) raise e finally: zip_ref.close() @@ -205,14 +207,25 @@ def add_app(self, installer): app.save() return app + def _rollback(self): + shutil.rmtree(self.app_dir) + apps_config = AppConfig() + app_conf = apps_config.get_by_name(self.name) + if app_conf: + del app_conf + def install(self, restart=True): self.upgrade = False if os.path.exists(self.app_dir): - installed_app = App.objects.get(name=self.name) - if installed_app.version < self.version["version"]: - self.upgrade = True - else: - raise AppAlreadyInstalledException() + try: + installed_app = App.objects.get(name=self.name) + if installed_app.version < self.version["version"]: + self.upgrade = True + else: + raise AppAlreadyInstalledException() + except App.DoesNotExist: + # NOTE:the following code handle if app downloaded and for some reason not added to the portal + self._rollback() installed_apps = [] for name, version in list(self.version["dependencies"].items()): # use try except because AppInstaller.__init__ will handle upgrade @@ -228,16 +241,24 @@ def install(self, restart=True): self.check_then_finlize(restart, installed_apps) return installed_apps + @transaction.atomic def check_then_finlize(self, restart, installed_apps): try: installer = importlib.import_module('%s.installer' % self.name) - installed_apps.append(self.add_app(installer)) - installer.install() + new_app = self.add_app(installer) + installed_apps.append(new_app) + try: + installer.install() + except Exception as e: + logger.error(e.message) FINALIZE_SETUP.apps_to_finlize.append(self.name) if restart: FINALIZE_SETUP(self.name) except ImportError as ex: logger.error(ex.message) + if os.path.exists(self.app_dir): + self._rollback() + raise ex def completely_remove(self): app = App.objects.get(name=self.name) diff --git a/cartoview/app_manager/static/app_manager/js/manager.js b/cartoview/app_manager/static/app_manager/js/manager.js index dc290397..c913d660 100644 --- a/cartoview/app_manager/static/app_manager/js/manager.js +++ b/cartoview/app_manager/static/app_manager/js/manager.js @@ -106,6 +106,9 @@ var _getDependents = function (app, appsHash, dependents) { angular.forEach(app.store.installedApps.objects, function (installedApp) { var currentApp = appsHash[installedApp.name]; + // NOTE:this line will raise an error while uninstalling app + // It was working fine before this commit in gitlab f976590bd981b05f062f055f1aed3040b190f8e9 + // I this our app market result for apps was changed without handle this change here if (dependents.indexOf(currentApp) == -1 && currentApp.latest_version.dependencies[app.name]) { dependents.push(currentApp) _getDependents(currentApp, appsHash, dependents) From b93ca1e230043d5c91898ada1ea31e20e74116e2 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 00:28:36 +0200 Subject: [PATCH 13/21] return js compatible version in check_version --- cartoview/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cartoview/views.py b/cartoview/views.py index a82c1164..cc382ffc 100644 --- a/cartoview/views.py +++ b/cartoview/views.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import requests from django.shortcuts import render +from pkg_resources import parse_version from .log_handler import get_logger from .version import get_current_version @@ -15,9 +16,13 @@ def index(request): def check_version(request): r = requests.get("https://pypi.org/pypi/cartoview/json") + _version = parse_version(get_current_version())._version + release = _version.release + version = [str(x) for x in release] + current_version = ".".join(version) context = dict( latest_version=r.json()["info"]["version"], - current_version=get_current_version()) + current_version=current_version) return render( request, "cartoview/check_version.js", From bcf3ce5732b53beba5b41f2ce7ea32b117898323 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 13:02:28 +0200 Subject: [PATCH 14/21] stop restarting in test --- cartoview/app_manager/installer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index 0d35bd3b..27d1cb76 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -6,6 +6,7 @@ import os import shutil import subprocess +import sys import tempfile import zipfile from builtins import * @@ -74,8 +75,9 @@ def _finalize_setup(app_name): self.docker_restart() else: self.restart_server() - timer = Timer(0.1, _finalize_setup(app_name)) - timer.start() + if 'test' not in sys.argv: + timer = Timer(0.1, _finalize_setup(app_name)) + timer.start() def __call__(self, app_name): self.finalize_setup(app_name) From 37a3cd8930b5d6d28f1d1992e79b7a70c4e364ce Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 13:04:17 +0200 Subject: [PATCH 15/21] thread locking in app installer --- cartoview/app_manager/installer.py | 93 +++++++++++++++++------------- 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index 27d1cb76..994716cc 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile +import threading import zipfile from builtins import * from io import BytesIO @@ -28,6 +29,7 @@ from .config import App as AppConfig from .helpers import change_path_permission, create_direcotry from .models import App, AppStore, AppType +from .settings import create_apps_dir logger = get_logger(__name__) install_app_batch = getattr(settings, 'INSTALL_APP_BAT', None) @@ -36,6 +38,8 @@ current_folder = os.path.abspath(os.path.dirname(__file__)) temp_dir = os.path.join(current_folder, 'temp') +lock = threading.Lock() + class FinalizeInstaller: def __init__(self): @@ -131,6 +135,7 @@ class AppAlreadyInstalledException(BaseException): class AppInstaller(object): def __init__(self, name, store_id=None, version=None, user=None): + create_apps_dir() self.user = user self.app_dir = os.path.join(settings.APPS_DIR, name) self.name = name @@ -217,31 +222,32 @@ def _rollback(self): del app_conf def install(self, restart=True): - self.upgrade = False - if os.path.exists(self.app_dir): - try: - installed_app = App.objects.get(name=self.name) - if installed_app.version < self.version["version"]: - self.upgrade = True - else: - raise AppAlreadyInstalledException() - except App.DoesNotExist: - # NOTE:the following code handle if app downloaded and for some reason not added to the portal - self._rollback() - installed_apps = [] - for name, version in list(self.version["dependencies"].items()): - # use try except because AppInstaller.__init__ will handle upgrade - # if version not match - try: - app_installer = AppInstaller( - name, self.store.id, version, user=self.user) - installed_apps += app_installer.install(restart=False) - except AppAlreadyInstalledException as e: - logger.error(e.message) - self._download_app() - reload(pkg_resources) - self.check_then_finlize(restart, installed_apps) - return installed_apps + with lock: + self.upgrade = False + if os.path.exists(self.app_dir): + try: + installed_app = App.objects.get(name=self.name) + if installed_app.version < self.version["version"]: + self.upgrade = True + else: + raise AppAlreadyInstalledException() + except App.DoesNotExist: + # NOTE:the following code handle if app downloaded and for some reason not added to the portal + self._rollback() + installed_apps = [] + for name, version in list(self.version["dependencies"].items()): + # use try except because AppInstaller.__init__ will handle upgrade + # if version not match + try: + app_installer = AppInstaller( + name, self.store.id, version, user=self.user) + installed_apps += app_installer.install(restart=False) + except AppAlreadyInstalledException as e: + logger.error(e.message) + self._download_app() + reload(pkg_resources) + self.check_then_finlize(restart, installed_apps) + return installed_apps @transaction.atomic def check_then_finlize(self, restart, installed_apps): @@ -274,7 +280,8 @@ def execute_command(self, command): process = subprocess.Popen("{} {} {}".format( executable, manage_py, command), shell=True) out, err = process.communicate() - logger.error(out, err) + logger.info(out) + logger.error(err) def delete_app_tables(self): from django.contrib.contenttypes.models import ContentType @@ -296,18 +303,22 @@ def uninstall(self, restart=True): :return: """ - installed_apps = App.objects.all() - for app in installed_apps: - app_installer = AppInstaller( - app.name, self.store.id, app.version, user=self.user) - dependencies = app_installer.version["dependencies"] - if self.name in dependencies: - app_installer.uninstall(restart=False) - installer = importlib.import_module('%s.installer' % self.name) - installer.uninstall() - self.delete_app_tables() - self.completely_remove() - app_dir = os.path.join(settings.APPS_DIR, self.name) - shutil.rmtree(app_dir) - if restart: - FINALIZE_SETUP.restart_server() + with lock: + uninstalled = False + installed_apps = App.objects.all() + for app in installed_apps: + app_installer = AppInstaller( + app.name, self.store.id, app.version, user=self.user) + dependencies = app_installer.version["dependencies"] + if self.name in dependencies: + app_installer.uninstall(restart=False) + installer = importlib.import_module('%s.installer' % self.name) + installer.uninstall() + self.delete_app_tables() + self.completely_remove() + app_dir = os.path.join(settings.APPS_DIR, self.name) + shutil.rmtree(app_dir) + uninstalled = True + if restart: + FINALIZE_SETUP.restart_server() + return uninstalled From ae3b73b11b7fb4e52cd48e115460fb8965f1e5d0 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 13:05:46 +0200 Subject: [PATCH 16/21] complete upgrade urls style to django 1.9+ compatible --- cartoview/app_manager/api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cartoview/app_manager/api.py b/cartoview/app_manager/api.py index 9abf3783..42874b3e 100644 --- a/cartoview/app_manager/api.py +++ b/cartoview/app_manager/api.py @@ -1,5 +1,6 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) + import warnings from builtins import * @@ -9,8 +10,11 @@ from tastypie.api import Api as TastypieApi from tastypie.utils import trailing_slash -from .serializers import HTMLSerializer +from cartoview.app_manager.api import home as app_manager_rest_home from cartoview.log_handler import get_logger + +from .serializers import HTMLSerializer + logger = get_logger(__name__) standard_library.install_aliases() @@ -86,7 +90,7 @@ def register(self, resource, app_name=None, canonical=True): def urls(self): pattern_list = [ url(r'^$', - 'cartoview.app_manager.api.home', + app_manager_rest_home, name='cartoview_rest_url'), ] for name in sorted(self.apis.keys()): From 5af3f396973888c59a2f00a71ca411e53d4c3b14 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 13:12:26 +0200 Subject: [PATCH 17/21] add unit testing part[1] --- cartoview/app_manager/settings.py | 7 +- cartoview/app_manager/tests/__init__.py | 0 cartoview/app_manager/tests/test_installer.py | 35 +++++++++ cartoview/app_manager/tests/test_views.py | 50 +++++++++++++ cartoview/app_manager/urls.py | 11 +-- cartoview/app_manager/views.py | 34 ++------- cartoview/cartoview_api/tests.py | 3 - cartoview/cartoview_api/tests/__init__.py | 0 cartoview/cartoview_api/tests/test_rest.py | 26 +++++++ cartoview/cartoview_api/tests/test_views.py | 22 ++++++ cartoview/settings.py | 10 ++- cartoview/tests/__init__.py | 0 cartoview/tests/test_log_handler.py | 12 +++ cartoview/tests/test_views.py | 75 +++++++++++++++++++ setup.py | 2 +- 15 files changed, 247 insertions(+), 40 deletions(-) create mode 100644 cartoview/app_manager/tests/__init__.py create mode 100644 cartoview/app_manager/tests/test_installer.py create mode 100644 cartoview/app_manager/tests/test_views.py delete mode 100644 cartoview/cartoview_api/tests.py create mode 100644 cartoview/cartoview_api/tests/__init__.py create mode 100644 cartoview/cartoview_api/tests/test_rest.py create mode 100644 cartoview/cartoview_api/tests/test_views.py create mode 100644 cartoview/tests/__init__.py create mode 100644 cartoview/tests/test_log_handler.py create mode 100644 cartoview/tests/test_views.py diff --git a/cartoview/app_manager/settings.py b/cartoview/app_manager/settings.py index a3984591..3d66eda1 100644 --- a/cartoview/app_manager/settings.py +++ b/cartoview/app_manager/settings.py @@ -19,12 +19,17 @@ # BASE_DIR must be defined in project.settings -def load_apps(): +def create_apps_dir(): from django.conf import settings if not os.path.exists(settings.APPS_DIR): create_direcotry(settings.APPS_DIR) if not os.access(settings.APPS_DIR, os.W_OK): change_path_permission(settings.APPS_DIR) + + +def load_apps(): + from django.conf import settings + create_apps_dir() if settings.APPS_DIR not in sys.path: sys.path.append(settings.APPS_DIR) diff --git a/cartoview/app_manager/tests/__init__.py b/cartoview/app_manager/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cartoview/app_manager/tests/test_installer.py b/cartoview/app_manager/tests/test_installer.py new file mode 100644 index 00000000..8dec67e2 --- /dev/null +++ b/cartoview/app_manager/tests/test_installer.py @@ -0,0 +1,35 @@ +from django.test import TestCase +from geonode.people.models import Profile +from cartoview.app_manager.models import AppStore +from cartoview.app_manager.installer import AppInstaller +from django.conf import settings +import shutil +import os +from cartoview.app_manager.helpers import create_direcotry +from cartoview.log_handler import get_logger +logger = get_logger(__name__) + + +class AppInstallerTest(TestCase): + fixtures = ['sample_admin.json'] + + def setUp(self): + AppStore.objects.create( + name="Test Store", url="https://appstore.cartoview.net/api/v1/", + is_default=True) + + def test_app_installer(self): + user = Profile.objects.filter(is_superuser=True).first() + store_id = 1 + app_name = "cartoview_basic_viewer" + app_version = "1.7.9" + installer = AppInstaller(app_name, store_id, app_version, user) + installed_apps = installer.install() + logger.debug(installed_apps) + self.assertEqual(len(installed_apps), 1) + uninstalled = installer.uninstall() + logger.debug(uninstalled) + self.assertEqual(uninstalled, True) + + def tearDown(self): + shutil.rmtree(settings.APPS_DIR) diff --git a/cartoview/app_manager/tests/test_views.py b/cartoview/app_manager/tests/test_views.py new file mode 100644 index 00000000..4128983c --- /dev/null +++ b/cartoview/app_manager/tests/test_views.py @@ -0,0 +1,50 @@ +from django.test import TestCase, RequestFactory +from django.core.urlresolvers import reverse +from geonode.people.models import Profile +from cartoview.app_manager.views import manage_apps + + +class CartoviewAppsViewTest(TestCase): + + def test_view_url_exists_at_desired_location(self): + resp = self.client.get('/apps/') + self.assertEqual(resp.status_code, 200) + + def test_view_url_accessible_by_name(self): + resp = self.client.get(reverse('app_manager_base_url')) + self.assertEqual(resp.status_code, 200) + + def test_view_uses_correct_template(self): + resp = self.client.get(reverse('app_manager_base_url')) + self.assertEqual(resp.status_code, 200) + self.assertTemplateUsed(resp, 'app_manager/apps.html') + self.assertTrue("apps" in resp.context) + + +class CartoviewManageAppsViewTest(TestCase): + fixtures = ['sample_admin.json'] + + def setUp(self): + self.factory = RequestFactory() + self.username = "staff_user" + self.password = "staff_user" + self.email = "staff_user@staff_user.com" + self.user = Profile.objects.create_superuser( + self.username, self.email, self.password) + self.client.login(username=self.username, password=self.password) + + def test_view_url_exists_at_desired_location(self): + resp = self.client.get("/apps/manage/") + self.assertEqual(resp.status_code, 200) + + def test_view_url_accessible_by_name(self): + resp = self.client.get(reverse("manage_apps")) + self.assertEqual(resp.status_code, 200) + + def test_view_uses_correct_template(self): + resp = self.client.get(reverse('manage_apps')) + self.assertEqual(resp.status_code, 200) + self.assertTemplateUsed(resp, 'app_manager/manage.html') + self.assertTrue("apps" in resp.context) + self.assertTrue("site_apps" in resp.context) + self.assertTrue("version_info" in resp.context) diff --git a/cartoview/app_manager/urls.py b/cartoview/app_manager/urls.py index cc83b677..f99c810f 100644 --- a/cartoview/app_manager/urls.py +++ b/cartoview/app_manager/urls.py @@ -87,9 +87,10 @@ def app_url(app_name): name=app + '_base_url') -apps_config = AppsConfig() -for app_config in apps_config: - import_app_rest(app_config.name) +def load_apps_urls(): + apps_config = AppsConfig() + for app_config in apps_config: + import_app_rest(app_config.name) -for app_config in apps_config: - urlpatterns.append(app_url(app_config.name)) + for app_config in apps_config: + urlpatterns.append(app_url(app_config.name)) diff --git a/cartoview/app_manager/views.py b/cartoview/app_manager/views.py index 1f124eb6..f437873d 100644 --- a/cartoview/app_manager/views.py +++ b/cartoview/app_manager/views.py @@ -7,14 +7,12 @@ import json import os from builtins import * -from urllib.parse import urljoin from django.conf import settings from django.conf.urls import url from django.contrib.admin.views.decorators import staff_member_required from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied -from django.core.files import File from django.core.urlresolvers import reverse from django.db import transaction from django.db.models import F, Max, Min @@ -64,32 +62,14 @@ os.makedirs(temp_dir) -def save_thumbnail(filename, image): - thumb_folder = 'thumbs' - upload_path = os.path.join(settings.MEDIA_ROOT, thumb_folder) - if not os.path.exists(upload_path): - os.makedirs(upload_path) - - with open(os.path.join(upload_path, filename), 'wb') as f: - thumbnail = File(f) - thumbnail.write(image) - - url_path = os.path.join( - settings.MEDIA_URL, - thumb_folder, - filename).replace( - '\\', - '/') - url = urljoin(settings.SITEURL, url_path) - - return url - - def get_apps_names(): - return [ - n for n in os.listdir(settings.APPS_DIR) - if os.path.isdir(os.path.join(settings.APPS_DIR, n)) - ] + apps = [] + if os.path.exists(settings.APPS_DIR): + apps = [ + n for n in os.listdir(settings.APPS_DIR) + if os.path.isdir(os.path.join(settings.APPS_DIR, n)) + ] + return apps def installed_apps(): diff --git a/cartoview/cartoview_api/tests.py b/cartoview/cartoview_api/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/cartoview/cartoview_api/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/cartoview/cartoview_api/tests/__init__.py b/cartoview/cartoview_api/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cartoview/cartoview_api/tests/test_rest.py b/cartoview/cartoview_api/tests/test_rest.py new file mode 100644 index 00000000..06a95224 --- /dev/null +++ b/cartoview/cartoview_api/tests/test_rest.py @@ -0,0 +1,26 @@ +from django.test import TestCase +from tastypie.test import ResourceTestCaseMixin +from django.core.urlresolvers import reverse +from cartoview.cartoview_api.rest import ExtendedResourceBaseResource +from geonode.urls import api + + +class ExtendedResourceBaseResourceTest(ResourceTestCaseMixin, TestCase): + fixtures = ['sample_admin.json'] + + def setUp(self): + super(ExtendedResourceBaseResourceTest, self).setUp() + self.username = 'admin' + self.password = 'admin' + + def get_credentials(self): + return self.create_basic(username=self.username, password=self.password) + + def test_get_list(self): + target_url = reverse( + "api_dispatch_list", + kwargs={"resource_name": + ExtendedResourceBaseResource.Meta.resource_name, + "api_name": api.api_name}) + self.assertHttpOK(self.api_client.get( + target_url, format='json')) diff --git a/cartoview/cartoview_api/tests/test_views.py b/cartoview/cartoview_api/tests/test_views.py new file mode 100644 index 00000000..9b8431f1 --- /dev/null +++ b/cartoview/cartoview_api/tests/test_views.py @@ -0,0 +1,22 @@ +from django.test import TestCase +from cartoview.cartoview_api.views import ( + get_geoserver_credintials, convert_infinty) +from django.conf import settings +from faker import Faker + + +class CartoviewApiViewsTest(TestCase): + def setUp(self): + self.faker = Faker() + + def test_get_geoserver_credintials(self): + gs_user, gs_password, gs_url = get_geoserver_credintials() + self.assertEqual(gs_user, settings.OGC_SERVER_DEFAULT_USER) + self.assertEqual(gs_password, settings.OGC_SERVER_DEFAULT_PASSWORD) + self.assertEqual(gs_url, settings.GEOSERVER_LOCATION) + + def test_convert_infinty(self): + self.assertIsNone(convert_infinty(float('inf'))) + self.assertIsNotNone(convert_infinty(self.faker.random.random())) + self.assertIsNotNone(convert_infinty(self.faker.name())) + self.assertIsNotNone(convert_infinty(self.faker.text())) diff --git a/cartoview/settings.py b/cartoview/settings.py index ccea9967..a84e2b92 100644 --- a/cartoview/settings.py +++ b/cartoview/settings.py @@ -7,7 +7,7 @@ import re import dj_database_url import cartoview - +import sys INSTALLED_APPS += ("cartoview", "cartoview.cartoview_api.apps.CartoviewAPIConfig", "cartoview.app_manager", "cartoview.site_management", @@ -61,7 +61,8 @@ from .local_settings import * except Exception as e: pass - +if 'test' in sys.argv: + DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', }} if 'datastore' in DATABASES: OGC_SERVER['default']['DATASTORE'] = 'datastore' @@ -74,4 +75,7 @@ CARTOVIEW_STAND_ALONE = strtobool(os.getenv('CARTOVIEW_STAND_ALONE', 'FALSE')) if CARTOVIEW_STAND_ALONE: from cartoview.app_manager.settings import load_apps - INSTALLED_APPS += load_apps() \ No newline at end of file + INSTALLED_APPS += load_apps() +if 'test' in sys.argv: + from cartoview.app_manager.settings import load_apps + INSTALLED_APPS += load_apps() diff --git a/cartoview/tests/__init__.py b/cartoview/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cartoview/tests/test_log_handler.py b/cartoview/tests/test_log_handler.py new file mode 100644 index 00000000..6eacb60c --- /dev/null +++ b/cartoview/tests/test_log_handler.py @@ -0,0 +1,12 @@ +from django.test import TestCase +from cartoview.log_handler import get_logger +from logging import Logger, DEBUG + + +class CartoviewHomeViewTest(TestCase): + + def test_get_logger(self): + logger = get_logger(__name__) + self.assertEqual(isinstance(logger, Logger), True) + self.assertEqual(logger.name, __name__) + self.assertEqual(logger.level, DEBUG) diff --git a/cartoview/tests/test_views.py b/cartoview/tests/test_views.py new file mode 100644 index 00000000..ff54dc62 --- /dev/null +++ b/cartoview/tests/test_views.py @@ -0,0 +1,75 @@ +from django.test import TestCase +from django.core.urlresolvers import reverse +from pkg_resources import parse_version +from cartoview.version import get_current_version +import requests +from geonode.urls import api +import json + + +class CartoviewHomeViewTest(TestCase): + + def test_view_url_exists_at_desired_location(self): + resp = self.client.get('/') + self.assertEqual(resp.status_code, 200) + + def test_view_url_accessible_by_name(self): + resp = self.client.get(reverse('home')) + self.assertEqual(resp.status_code, 200) + + def test_view_uses_correct_template(self): + resp = self.client.get(reverse('home')) + self.assertEqual(resp.status_code, 200) + self.assertTemplateUsed(resp, 'site_index.html') + + +class CartoviewCheckVersionViewTest(TestCase): + + def test_view_url_exists_at_desired_location(self): + resp = self.client.get('/check-version/') + self.assertEqual(resp.status_code, 200) + + def test_view_url_accessible_by_name(self): + resp = self.client.get(reverse('check_version')) + self.assertEqual(resp.status_code, 200) + + def test_view_uses_correct_template(self): + resp = self.client.get(reverse('check_version')) + self.assertEqual(resp.status_code, 200) + self.assertTemplateUsed(resp, 'cartoview/check_version.js') + self.assertEqual("current_version" in resp.context, True) + self.assertEqual("latest_version" in resp.context, True) + _version = parse_version(get_current_version())._version + release = _version.release + version = [str(x) for x in release] + current_version = ".".join(version) + req = requests.get("https://pypi.org/pypi/cartoview/json") + latest_version = str(req.json()["info"]["version"]) + self.assertEqual(resp.context["current_version"], current_version) + self.assertEqual(resp.context["latest_version"], latest_version) + + +class CartoviewResourcesRegisteredTest(TestCase): + + def test_api_url_exists_at_desired_location(self): + resp = self.client.get('/api/') + self.assertEqual(resp.status_code, 200) + + def test_api_url_accessible_by_name(self): + resp = self.client.get( + reverse("api_%s_top_level" % api.api_name, + kwargs={"api_name": api.api_name})) + self.assertEqual(resp.status_code, 200) + + def test_cartoview_resources_exists(self): + resp = self.client.get( + reverse("api_%s_top_level" % api.api_name, + kwargs={"api_name": api.api_name})) + self.assertEqual(resp.status_code, 200) + resources_dict = json.loads(resp.content) + self.assertTrue("app" in resources_dict.keys()) + self.assertTrue("appinstances" in resources_dict.keys()) + self.assertTrue("all_resources" in resources_dict.keys()) + for endpoint in resources_dict.values(): + resp = self.client.get(endpoint['list_endpoint']) + self.assertEqual(resp.status_code, 200) diff --git a/setup.py b/setup.py index efb99fb3..6facc221 100644 --- a/setup.py +++ b/setup.py @@ -31,4 +31,4 @@ "Topic :: Scientific/Engineering :: GIS"], license="BSD", install_requires=['future', 'six==1.10.0', 'portalocker==1.2.1', - 'geonode>=2.8rc11']) + 'geonode>2.7.5.dev20180125135927', 'Faker>=0.8.4']) From 0ec23fef8b9a369ab8a70fffaf4cdff0ae806b34 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 13:34:20 +0200 Subject: [PATCH 18/21] add travis.yml --- .travis.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..297feb11 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,48 @@ +sudo: required + +language: python + +cache: + directories: + - $HOME/.cache/pip + +before_cache: + - rm -f $HOME/.cache/pip/log/debug.log + +python: + - "2.7" + +virtualenv: + system_site_packages: true + +branches: + only: + - master + - stand_alone + +install: + - sudo apt-get -qq -y update + - sudo apt-get install software-properties-common python-software-properties -y + - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable + - sudo apt-get -qq -y update + - sudo apt-get upgrade -y + - sudo apt-get install -y libgdal1h python-gdal gdal-bin spatialite-bin + - sudo apt-get install -y python-virtualenv python-imaging python-lxml python-pyproj python-shapely python-nose python-httplib2 python-httplib2 gettext + - sudo apt-get install -y python-dev libxml2 libxml2-dev libxslt1-dev zlib1g-dev libjpeg-dev libpq-dev libgdal-dev git default-jdk + - sudo add-apt-repository -y ppa:webupd8team/java + - sudo apt-get update + - sudo apt-get install -y --force-yes oracle-java8-installer ant maven2 --no-install-recommends + - sudo update-java-alternatives --set java-8-oracle + - pip install -e . --upgrade + - pip install pygdal==`gdal-config --version` + - pip install coveralls + +script: + - coverage run --source=cartoview --omit="*/migrations/*,*/apps/*" ./manage.py test + +after_success: + - coveralls + +notifications: + email: + - hisham.karam@cartologic.com \ No newline at end of file From 756bbbb6a8b03385a975ce2df7baeec73a51773a Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 13:36:52 +0200 Subject: [PATCH 19/21] update travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 297feb11..7b156409 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,4 +45,5 @@ after_success: notifications: email: - - hisham.karam@cartologic.com \ No newline at end of file + - hisham.karam@cartologic.com + - hishamwaleedkaram@gmail.com \ No newline at end of file From 888ffe03afdcf315456f4d66f03bfa1cd49e2ae3 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 13:48:00 +0200 Subject: [PATCH 20/21] fix postgres error --- .travis.yml | 15 ++++++++++----- setup.py | 1 - 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b156409..345c8262 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,19 +22,24 @@ branches: install: - sudo apt-get -qq -y update + - sudo apt-get purge postgresql* -y - sudo apt-get install software-properties-common python-software-properties -y - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable - sudo apt-get -qq -y update - - sudo apt-get upgrade -y - - sudo apt-get install -y libgdal1h python-gdal gdal-bin spatialite-bin - - sudo apt-get install -y python-virtualenv python-imaging python-lxml python-pyproj python-shapely python-nose python-httplib2 python-httplib2 gettext - - sudo apt-get install -y python-dev libxml2 libxml2-dev libxslt1-dev zlib1g-dev libjpeg-dev libpq-dev libgdal-dev git default-jdk + - sudo apt-get upgrade -y --allow-unauthenticated + - sudo apt-get install -y gcc gettext python-pip libpq-dev sqlite3 git gdal-bin lsof psmisc + - sudo apt-get install -y python-gdal python-psycopg2 python-imaging python-lxml + - sudo apt-get install -y python-dev libgdal-dev libgeoip-dev python-ldap libxml2 libxml2-dev libxslt-dev git default-jdk + - sudo apt-get install -y libmemcached-dev libsasl2-dev zlib1g-dev python-pylibmc python-setuptools + - sudo apt-get install -y curl build-essential build-essential python-dev libffi-dev libssl-dev - sudo add-apt-repository -y ppa:webupd8team/java - sudo apt-get update - sudo apt-get install -y --force-yes oracle-java8-installer ant maven2 --no-install-recommends - sudo update-java-alternatives --set java-8-oracle + - pip install celery + - pip install geonode>2.7.5.dev20180125135927 - pip install -e . --upgrade - - pip install pygdal==`gdal-config --version` + - pip install django-autocomplete-light==2.3.3 - pip install coveralls script: diff --git a/setup.py b/setup.py index 6facc221..8d9ab08a 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,6 @@ author_email='info@cartologic.com', url='https://github.com/cartologic/cartoview', include_package_data=True, - python_requires='>=2.7.12, !=3.0.*, !=3.1.*, !=3.2.*, <4', keywords=[ 'cartoview', 'gis', From f2a367ef9a36424e6b8dc9630762dc0c5a1ffd97 Mon Sep 17 00:00:00 2001 From: hisham waleed karam Date: Sun, 5 Aug 2018 15:36:36 +0200 Subject: [PATCH 21/21] fix error ImportError: cannot import name home --- cartoview/app_manager/api.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cartoview/app_manager/api.py b/cartoview/app_manager/api.py index 42874b3e..522d3b35 100644 --- a/cartoview/app_manager/api.py +++ b/cartoview/app_manager/api.py @@ -1,6 +1,5 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) - import warnings from builtins import * @@ -9,16 +8,17 @@ from future import standard_library from tastypie.api import Api as TastypieApi from tastypie.utils import trailing_slash - -from cartoview.app_manager.api import home as app_manager_rest_home -from cartoview.log_handler import get_logger - from .serializers import HTMLSerializer - +from cartoview.log_handler import get_logger logger = get_logger(__name__) standard_library.install_aliases() +def home(request): + return render(request, 'app_manager/rest_api/home.html', + {'apis': sorted(rest_api.apis.keys())}) + + class BaseApi(TastypieApi): """ A version of the Api that doesn't require a name. @@ -90,7 +90,7 @@ def register(self, resource, app_name=None, canonical=True): def urls(self): pattern_list = [ url(r'^$', - app_manager_rest_home, + home, name='cartoview_rest_url'), ] for name in sorted(self.apis.keys()): @@ -105,8 +105,3 @@ def register_app_urls(self, app_name): rest_api = Api() - - -def home(request): - return render(request, 'app_manager/rest_api/home.html', - {'apis': sorted(rest_api.apis.keys())})