diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..345c8262 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,54 @@ +sudo: required + +language: python + +cache: + directories: + - $HOME/.cache/pip + +before_cache: + - rm -f $HOME/.cache/pip/log/debug.log + +python: + - "2.7" + +virtualenv: + system_site_packages: true + +branches: + only: + - master + - stand_alone + +install: + - sudo apt-get -qq -y update + - sudo apt-get purge postgresql* -y + - sudo apt-get install software-properties-common python-software-properties -y + - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable + - sudo apt-get -qq -y update + - sudo apt-get upgrade -y --allow-unauthenticated + - sudo apt-get install -y gcc gettext python-pip libpq-dev sqlite3 git gdal-bin lsof psmisc + - sudo apt-get install -y python-gdal python-psycopg2 python-imaging python-lxml + - sudo apt-get install -y python-dev libgdal-dev libgeoip-dev python-ldap libxml2 libxml2-dev libxslt-dev git default-jdk + - sudo apt-get install -y libmemcached-dev libsasl2-dev zlib1g-dev python-pylibmc python-setuptools + - sudo apt-get install -y curl build-essential build-essential python-dev libffi-dev libssl-dev + - sudo add-apt-repository -y ppa:webupd8team/java + - sudo apt-get update + - sudo apt-get install -y --force-yes oracle-java8-installer ant maven2 --no-install-recommends + - sudo update-java-alternatives --set java-8-oracle + - pip install celery + - pip install geonode>2.7.5.dev20180125135927 + - pip install -e . --upgrade + - pip install django-autocomplete-light==2.3.3 + - pip install coveralls + +script: + - coverage run --source=cartoview --omit="*/migrations/*,*/apps/*" ./manage.py test + +after_success: + - coveralls + +notifications: + email: + - hisham.karam@cartologic.com + - hishamwaleedkaram@gmail.com \ No newline at end of file diff --git a/README.md b/README.md index d325ee1f..58eafdb2 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ ``` - open your browser and type the following address `10.5.0.4` - default user credentials `admin/admin` for cartoview and `admin/geoserver` for geoserver + - you need to configure oauth in geonode and geoserver to do this please use this [link](http://docs.geonode.org/en/master/tutorials/admin/geoserver_geonode_security/index.html) - you can stop containers with `make down` or `docker-compose down` - you can get logs for each service in `docker-compose.yml` unsing the following command: - `docker-compose logs --follow --tail=100 ` @@ -65,5 +66,6 @@ INSTALLED_APPS += load_apps() ``` - restart your server + ## Docs: - [How to use and install](http://cartologic.github.io) diff --git a/cartoview/__init__.py b/cartoview/__init__.py index 3f7699ab..4eb2d954 100644 --- a/cartoview/__init__.py +++ b/cartoview/__init__.py @@ -1,6 +1,6 @@ from .celery import app -__version__ = (1, 8, 2, 'rc', 0) +__version__ = (1, 8, 2, 'rc', 1) __compatible_with__ = [] diff --git a/cartoview/app_manager/api.py b/cartoview/app_manager/api.py index 75c5637e..522d3b35 100644 --- a/cartoview/app_manager/api.py +++ b/cartoview/app_manager/api.py @@ -3,18 +3,22 @@ import warnings from builtins import * -from django.conf.urls import include, patterns, url +from django.conf.urls import include, url from django.shortcuts import render from future import standard_library from tastypie.api import Api as TastypieApi from tastypie.utils import trailing_slash - from .serializers import HTMLSerializer from cartoview.log_handler import get_logger logger = get_logger(__name__) standard_library.install_aliases() +def home(request): + return render(request, 'app_manager/rest_api/home.html', + {'apis': sorted(rest_api.apis.keys())}) + + class BaseApi(TastypieApi): """ A version of the Api that doesn't require a name. @@ -47,7 +51,8 @@ def urls(self): for name in sorted(self._registry.keys()): resource = self._registry[name] resource.api_name = self.api_name - pattern_list.append((r"^%s" % api_pattern, include(resource.urls))) + pattern_list.append( + url(r"^%s" % api_pattern, include(resource.urls))) urlpatterns = self.prepend_urls() overridden_urls = self.override_urls() @@ -60,7 +65,7 @@ def urls(self): urlpatterns += overridden_urls - urlpatterns += patterns('', *pattern_list) + urlpatterns += pattern_list return urlpatterns @@ -85,13 +90,13 @@ def register(self, resource, app_name=None, canonical=True): def urls(self): pattern_list = [ url(r'^$', - 'cartoview.app_manager.api.home', + home, name='cartoview_rest_url'), ] for name in sorted(self.apis.keys()): pattern_list.append( url(r"^%s/" % name, include(self.apis[name].urls))) - self.urlpatterns = patterns('', *pattern_list) + self.urlpatterns = pattern_list return self.urlpatterns def register_app_urls(self, app_name): @@ -100,8 +105,3 @@ def register_app_urls(self, app_name): rest_api = Api() - - -def home(request): - return render(request, 'app_manager/rest_api/home.html', - {'apis': sorted(rest_api.apis.keys())}) diff --git a/cartoview/app_manager/helpers.py b/cartoview/app_manager/helpers.py index 6a6f376a..71c8a875 100644 --- a/cartoview/app_manager/helpers.py +++ b/cartoview/app_manager/helpers.py @@ -1,6 +1,8 @@ import os import stat world_permission = 0o777 +from cartoview.log_handler import get_logger +logger = get_logger(__name__) def create_direcotry(path, mode=0777): @@ -10,6 +12,8 @@ def create_direcotry(path, mode=0777): try: previous_mask = os.umask(0) os.makedirs(path, mode=mode) + except OSError as e: + logger.error(e.message) finally: # set the previous mask back os.umask(previous_mask) diff --git a/cartoview/app_manager/installer.py b/cartoview/app_manager/installer.py index cbd62af4..994716cc 100644 --- a/cartoview/app_manager/installer.py +++ b/cartoview/app_manager/installer.py @@ -3,32 +3,34 @@ unicode_literals) import importlib -import logging import os import shutil import subprocess +import sys import tempfile +import threading import zipfile -import yaml from builtins import * from io import BytesIO -from .helpers import create_direcotry, change_path_permission -from sys import stdout, exit, executable +from sys import executable, exit from threading import Timer + import pkg_resources +import portalocker import requests +import yaml from django.conf import settings +from django.db import transaction from django.db.models import Max from future import standard_library +from cartoview.log_handler import get_logger + from .config import App as AppConfig -# TODO: find a cross platform function (fcntl is not supported by windows) -try: - import fcntl -except Exception: - pass +from .helpers import change_path_permission, create_direcotry from .models import App, AppStore, AppType -from cartoview.log_handler import get_logger +from .settings import create_apps_dir + logger = get_logger(__name__) install_app_batch = getattr(settings, 'INSTALL_APP_BAT', None) standard_library.install_aliases() @@ -36,6 +38,8 @@ current_folder = os.path.abspath(os.path.dirname(__file__)) temp_dir = os.path.join(current_folder, 'temp') +lock = threading.Lock() + class FinalizeInstaller: def __init__(self): @@ -43,14 +47,8 @@ def __init__(self): def save_pending_app_to_finlize(self): with open(settings.PENDING_APPS, 'wb') as outfile: - if 'fcntl' in globals(): - fcntl.flock(outfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - yaml.dump(self.apps_to_finlize, outfile, - default_flow_style=False) - fcntl.flock(outfile, fcntl.LOCK_UN) - else: - yaml.dump(self.apps_to_finlize, outfile, - default_flow_style=False) + portalocker.lock(outfile, portalocker.LOCK_EX) + yaml.dump(self.apps_to_finlize, outfile, default_flow_style=False) self.apps_to_finlize = [] def restart_server(self): @@ -81,8 +79,9 @@ def _finalize_setup(app_name): self.docker_restart() else: self.restart_server() - timer = Timer(0.1, _finalize_setup(app_name)) - timer.start() + if 'test' not in sys.argv: + timer = Timer(0.1, _finalize_setup(app_name)) + timer.start() def __call__(self, app_name): self.finalize_setup(app_name) @@ -136,6 +135,7 @@ class AppAlreadyInstalledException(BaseException): class AppInstaller(object): def __init__(self, name, store_id=None, version=None, user=None): + create_apps_dir() self.user = user self.app_dir = os.path.join(settings.APPS_DIR, name) self.name = name @@ -157,13 +157,9 @@ def _request_rest_data(self, *args): """ get app information form app store rest url """ - try: - q = requests.get(self.store.url + ''.join( - [str(item) for item in args])) - return q.json() - except Exception as e: - logger.error(e.message) - return None + q = requests.get(self.store.url + ''.join( + [str(item) for item in args])) + return q.json() def extract_move_app(self, zipped_app): extract_to = tempfile.mkdtemp(dir=temp_dir) @@ -188,7 +184,8 @@ def _download_app(self): if not os.access(temp_dir, os.W_OK): change_path_permission(temp_dir) self.extract_move_app(zip_ref) - except Exception as e: + except shutil.Error as e: + logger.error(e.message) raise e finally: zip_ref.close() @@ -217,39 +214,59 @@ def add_app(self, installer): app.save() return app - def install(self, restart=True): - self.upgrade = False - if os.path.exists(self.app_dir): - installed_app = App.objects.get(name=self.name) - if installed_app.version < self.version["version"]: - self.upgrade = True - else: - raise AppAlreadyInstalledException() - installed_apps = [] - for name, version in list(self.version["dependencies"].items()): - # use try except because AppInstaller.__init__ will handle upgrade - # if version not match - try: - app_installer = AppInstaller( - name, self.store.id, version, user=self.user) - installed_apps += app_installer.install(restart=False) - except AppAlreadyInstalledException as e: - logger.error(e.message) - self._download_app() - reload(pkg_resources) - self.check_then_finlize(restart, installed_apps) - return installed_apps + def _rollback(self): + shutil.rmtree(self.app_dir) + apps_config = AppConfig() + app_conf = apps_config.get_by_name(self.name) + if app_conf: + del app_conf + def install(self, restart=True): + with lock: + self.upgrade = False + if os.path.exists(self.app_dir): + try: + installed_app = App.objects.get(name=self.name) + if installed_app.version < self.version["version"]: + self.upgrade = True + else: + raise AppAlreadyInstalledException() + except App.DoesNotExist: + # NOTE:the following code handle if app downloaded and for some reason not added to the portal + self._rollback() + installed_apps = [] + for name, version in list(self.version["dependencies"].items()): + # use try except because AppInstaller.__init__ will handle upgrade + # if version not match + try: + app_installer = AppInstaller( + name, self.store.id, version, user=self.user) + installed_apps += app_installer.install(restart=False) + except AppAlreadyInstalledException as e: + logger.error(e.message) + self._download_app() + reload(pkg_resources) + self.check_then_finlize(restart, installed_apps) + return installed_apps + + @transaction.atomic def check_then_finlize(self, restart, installed_apps): try: installer = importlib.import_module('%s.installer' % self.name) - installed_apps.append(self.add_app(installer)) - installer.install() + new_app = self.add_app(installer) + installed_apps.append(new_app) + try: + installer.install() + except Exception as e: + logger.error(e.message) FINALIZE_SETUP.apps_to_finlize.append(self.name) if restart: FINALIZE_SETUP(self.name) except ImportError as ex: logger.error(ex.message) + if os.path.exists(self.app_dir): + self._rollback() + raise ex def completely_remove(self): app = App.objects.get(name=self.name) @@ -263,7 +280,8 @@ def execute_command(self, command): process = subprocess.Popen("{} {} {}".format( executable, manage_py, command), shell=True) out, err = process.communicate() - logger.error(out, err) + logger.info(out) + logger.error(err) def delete_app_tables(self): from django.contrib.contenttypes.models import ContentType @@ -285,18 +303,22 @@ def uninstall(self, restart=True): :return: """ - installed_apps = App.objects.all() - for app in installed_apps: - app_installer = AppInstaller( - app.name, self.store.id, app.version, user=self.user) - dependencies = app_installer.version["dependencies"] - if self.name in dependencies: - app_installer.uninstall(restart=False) - installer = importlib.import_module('%s.installer' % self.name) - installer.uninstall() - self.delete_app_tables() - self.completely_remove() - app_dir = os.path.join(settings.APPS_DIR, self.name) - shutil.rmtree(app_dir) - if restart: - FINALIZE_SETUP.restart_server() + with lock: + uninstalled = False + installed_apps = App.objects.all() + for app in installed_apps: + app_installer = AppInstaller( + app.name, self.store.id, app.version, user=self.user) + dependencies = app_installer.version["dependencies"] + if self.name in dependencies: + app_installer.uninstall(restart=False) + installer = importlib.import_module('%s.installer' % self.name) + installer.uninstall() + self.delete_app_tables() + self.completely_remove() + app_dir = os.path.join(settings.APPS_DIR, self.name) + shutil.rmtree(app_dir) + uninstalled = True + if restart: + FINALIZE_SETUP.restart_server() + return uninstalled diff --git a/cartoview/app_manager/models.py b/cartoview/app_manager/models.py index b14cbc1e..6baa4e00 100644 --- a/cartoview/app_manager/models.py +++ b/cartoview/app_manager/models.py @@ -2,25 +2,26 @@ unicode_literals) import json -import logging from builtins import * -from sys import stdout from datetime import datetime -from django.template.defaultfilters import slugify + from django.conf import settings as geonode_settings from django.contrib.gis.db import models from django.core.urlresolvers import reverse from django.db.models import signals +from django.template.defaultfilters import slugify from future import standard_library -from jsonfield import JSONField -from taggit.managers import TaggableManager # Create your models here. from geonode.base.models import ResourceBase, resourcebase_post_save from geonode.maps.models import Map as GeonodeMap -from geonode.security.models import (remove_object_permissions) +from geonode.security.models import remove_object_permissions +from jsonfield import JSONField +from taggit.managers import TaggableManager -from .config import AppsConfig from cartoview.log_handler import get_logger + +from .config import AppsConfig + logger = get_logger(__name__) standard_library.install_aliases() @@ -161,7 +162,7 @@ def name_long(self): return str(self.id) else: return '%s (%s)' % (self.title, self.id) - + @property def config_obj(self): try: diff --git a/cartoview/app_manager/rest.py b/cartoview/app_manager/rest.py index a3368f0d..41abe8fe 100644 --- a/cartoview/app_manager/rest.py +++ b/cartoview/app_manager/rest.py @@ -3,7 +3,7 @@ unicode_literals) import json -from cartoview.app_manager.models import App, AppInstance, AppStore, AppType + from django.conf import settings from django.conf.urls import url from django.core.exceptions import ObjectDoesNotExist @@ -11,7 +11,8 @@ from future import standard_library from geonode.api.api import ProfileResource from geonode.api.authorization import GeoNodeAuthorization -from geonode.api.resourcebase_api import CommonMetaApi +from geonode.api.resourcebase_api import (CommonMetaApi, LayerResource, + MapResource) from geonode.layers.models import Attribute from geonode.maps.models import MapLayer from geonode.people.models import Profile @@ -23,8 +24,11 @@ from tastypie.resources import ModelResource from tastypie.utils import trailing_slash +from cartoview.app_manager.models import App, AppInstance, AppStore, AppType + from .resources import FileUploadResource -from geonode.api.resourcebase_api import LayerResource, MapResource +from .utils import populate_apps + standard_library.install_aliases() @@ -149,6 +153,7 @@ def set_active(self, active, request, **kwargs): bundle=bundle, **self.remove_api_resource_names(kwargs)) app.config.active = active app.apps_config.save() + populate_apps() except ObjectDoesNotExist: return HttpGone() @@ -207,6 +212,19 @@ class Meta(CommonMetaApi): excludes = ['csw_anytext', 'metadata_xml'] authorization = GeoNodeAuthorization() + def get_object_list(self, request): + __inactive_apps = [ + app.id for app in App.objects.all() if not app.config.active] + __inactive_apps_instances = [instance.id for instance in + AppInstance.objects.filter( + app__id__in=__inactive_apps)] + active_app_instances = super(AppInstanceResource, self)\ + .get_object_list( + request).exclude( + id__in=__inactive_apps_instances) + + return active_app_instances + def dehydrate_owner(self, bundle): return bundle.obj.owner.username diff --git a/cartoview/app_manager/settings.py b/cartoview/app_manager/settings.py index a3984591..3d66eda1 100644 --- a/cartoview/app_manager/settings.py +++ b/cartoview/app_manager/settings.py @@ -19,12 +19,17 @@ # BASE_DIR must be defined in project.settings -def load_apps(): +def create_apps_dir(): from django.conf import settings if not os.path.exists(settings.APPS_DIR): create_direcotry(settings.APPS_DIR) if not os.access(settings.APPS_DIR, os.W_OK): change_path_permission(settings.APPS_DIR) + + +def load_apps(): + from django.conf import settings + create_apps_dir() if settings.APPS_DIR not in sys.path: sys.path.append(settings.APPS_DIR) diff --git a/cartoview/app_manager/static/app_manager/js/manager.js b/cartoview/app_manager/static/app_manager/js/manager.js index dc290397..c913d660 100644 --- a/cartoview/app_manager/static/app_manager/js/manager.js +++ b/cartoview/app_manager/static/app_manager/js/manager.js @@ -106,6 +106,9 @@ var _getDependents = function (app, appsHash, dependents) { angular.forEach(app.store.installedApps.objects, function (installedApp) { var currentApp = appsHash[installedApp.name]; + // NOTE:this line will raise an error while uninstalling app + // It was working fine before this commit in gitlab f976590bd981b05f062f055f1aed3040b190f8e9 + // I this our app market result for apps was changed without handle this change here if (dependents.indexOf(currentApp) == -1 && currentApp.latest_version.dependencies[app.name]) { dependents.push(currentApp) _getDependents(currentApp, appsHash, dependents) diff --git a/cartoview/app_manager/templates/app_instance/_search_content.html b/cartoview/app_manager/templates/app_instance/_search_content.html index 0c742e67..a07c2ba0 100644 --- a/cartoview/app_manager/templates/app_instance/_search_content.html +++ b/cartoview/app_manager/templates/app_instance/_search_content.html @@ -25,6 +25,22 @@
{% include "search/_general_filters.html" %}
+
+ +
diff --git a/cartoview/app_manager/templates/app_manager/app_instance_list.html b/cartoview/app_manager/templates/app_manager/app_instance_list.html index 47d2f56c..3775f8ff 100644 --- a/cartoview/app_manager/templates/app_manager/app_instance_list.html +++ b/cartoview/app_manager/templates/app_manager/app_instance_list.html @@ -4,7 +4,10 @@ {% load url from future %} {% block title %} {% trans "Explore Apps" %} - {{ block.super }} {% endblock %} - +{% block head %} + {{ block.super }} + +{% endblock %} {% block body_class %}appinstances appinstances-list explore{% endblock %} {% block body %} @@ -47,8 +50,10 @@

{% verbatim %}{{ appTitle }}{% endverbatim %}

diff --git a/cartoview/app_manager/templates/app_manager/manage.html b/cartoview/app_manager/templates/app_manager/manage.html index 70266d9c..eed0de6c 100644 --- a/cartoview/app_manager/templates/app_manager/manage.html +++ b/cartoview/app_manager/templates/app_manager/manage.html @@ -165,6 +165,7 @@