Merge branch 'synchronization' into 0.4

.travis.yml

@@ -41,6 +41,7 @@ script:
   - python manage.py test backends -v2
   - python manage.py test coverages -v2
   - python manage.py test autotest_services -v2
+  - python manage.py test autotest_coverages -v2
 notifications:
   irc:
     channels:

eoxserver/backends/interfaces.py

@@ -103,12 +103,12 @@ def extract(self, package_filename, location, path):
                       this can be different than the passed ``path``
         """
 
-    def list_contents(self, package_filename, location):
+    def list_contents(self, package_filename, location_regex=None):
         """ Return a list of item locations under the specified location in the
             given package.
 
             :param package_filename: the local filename of the package
-            :param location: a template to find items within the package
+            :param location_regex: a template to find items within the package
             :returns: an iterable of the package contents under the specified
                       ``location``
         """

eoxserver/backends/packages/zip.py

@@ -28,6 +28,7 @@
 
 import shutil
 from zipfile import ZipFile
+import re
 
 from eoxserver.core import Component, implements
 from eoxserver.backends.interfaces import PackageInterface
@@ -36,7 +37,7 @@
 class ZIPPackage(Component):
     """Implementation of the package interface for ZIP package files.
     """
-    
+
     implements(PackageInterface)
 
     name = "ZIP"
@@ -47,7 +48,9 @@ def extract(self, package_filename, location, path):
         with open(path, "wb") as outfile:
             shutil.copyfileobj(infile, outfile)
 
-
-    def list_files(self, package_filename):
+    def list_files(self, package_filename, location_regex=None):
         zipfile = ZipFile(package_filename, "r")
-        # TODO: get list
+        filenames = zipfile.namelist()
+        if location_regex:
+            filenames = [f for f in filenames if re.match(location_regex, f)]
+        return filenames

eoxserver/backends/storages/local.py

@@ -27,17 +27,25 @@
 
 
 import os.path
-from urlparse import urlparse
+from glob import glob
 
 from eoxserver.core import Component, implements
 from eoxserver.backends.interfaces import FileStorageInterface
 
 
 class LocalStorage(Component):
-    implements(FileStorageInterface)
+    """ Implementation of the
+        :class:`eoxserver.backends.interfaces.FileStorageInterface` for local
+        storages.
+    """
 
+    implements(FileStorageInterface)
 
     name = "local"
 
     def retrieve(self, url, location, path):
         return location
+
+    def list_files(self, url, location_regex=None):
+        location_regex = location_regex or "*"
+        return glob(os.path.join(url, location_regex))

eoxserver/instance_template/project_name/settings.py

@@ -182,6 +182,8 @@
     'django.contrib.admindocs',
     # Enable the databrowse:
     #'django.contrib.databrowse',
+    # Enable for better schema and data-migrations
+    #'south',
     # Enable for debugging
     #'django_extensions',
     # Enable EOxServer:
@@ -193,7 +195,8 @@
     'eoxserver.testing',
     'eoxserver.webclient',
     # Enable EOxServer autotests
-    #'autotest_services',
+    'autotest_services',
+    'autotest_coverages',
 )
 
 
@@ -212,6 +215,9 @@
     # metadata readers/writers
     'eoxserver.resources.coverages.metadata.formats.*',
 
+	# registration schemes
+    'eoxserver.resources.coverages.registration.registrators.*',
+
     # service handlers
     'eoxserver.services.ows.wcs.**',
     'eoxserver.services.ows.wms.**',

eoxserver/resources/coverages/management/commands/eoxs_collection_synchronize.py

@@ -0,0 +1,133 @@
+#-------------------------------------------------------------------------------
+#
+# Project: EOxServer <http://eoxserver.org>
+# Authors: Fabian Schindler <fabian.schindler@eox.at>
+#
+#-------------------------------------------------------------------------------
+# Copyright (C) 2014 EOX IT Services GmbH
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#-------------------------------------------------------------------------------
+
+from optparse import make_option
+from itertools import product
+
+from django.core.management.base import CommandError, BaseCommand
+
+from eoxserver.resources.coverages import models
+from eoxserver.resources.coverages.synchronization import synchronize
+from eoxserver.resources.coverages.management.commands import (
+    CommandOutputMixIn, _variable_args_cb, nested_commit_on_success
+)
+
+
+class Command(CommandOutputMixIn, BaseCommand):
+    option_list = BaseCommand.option_list + (
+        make_option("identifier", dest="collection_ids",
+            action='callback', callback=_variable_args_cb,
+            default=None, help=("Collection(s) from which the "
+                                "objects shall be removed.")
+        ),
+    )
+
+    args = (
+        "<collection-id> [<collection-id> ...] "
+    )
+
+    help = """
+        Synchronizes one or more collections and all their data sources.
+    """
+
+
+    @nested_commit_on_success
+    def handle(self, *args, **kwargs):
+
+
+        synchronize()
+
+
+
+        # check the required inputs
+        collection_ids = kwargs.get('collection_ids', None)
+        remove_ids = kwargs.get('remove_ids', None)
+        if not collection_ids: 
+            raise CommandError(
+                "Missing the mandatory collection identifier(s)!"
+            )
+
+        if not remove_ids: 
+            raise CommandError(
+                "Missing the mandatory identifier(s) for to be removed "
+                "objects."
+            )
+
+        # extract the collections 
+        ignore_missing_collection = kwargs['ignore_missing_collection']
+        collections = [] 
+        for collection_id in collection_ids: 
+            try: 
+                collections.append(
+                    models.Collection.objects.get(identifier=collection_id)
+                )
+            except models.Collection.DoesNotExist: 
+                msg = (
+                    "There is no Collection matching the given "
+                    "identifier: '%s'" % collection_id
+                )
+                if ignore_missing_collection: 
+                    self.print_wrn(msg)
+                else: 
+                    raise CommandError(msg) 
+
+        # extract the children  
+        ignore_missing_object = kwargs['ignore_missing_object']
+        objects = [] 
+        for remove_id in remove_ids: 
+            try:
+                objects.append(
+                    models.EOObject.objects.get(identifier=remove_id)
+                )
+            except models.EOObject.DoesNotExist:
+                msg = (
+                    "There is no EOObject matching the given identifier: '%s'"
+                    % remove_id
+                )
+                if ignore_missing_object:
+                    self.print_wrn(msg)
+                else:
+                    raise CommandError(msg)
+
+        try:
+            for collection, eo_object in product(collections, objects):
+                # check whether the link does not exist
+                if eo_object in collection:
+                    self.print_msg(
+                        "Unlinking: %s <-x- %s" % (collection, eo_object)
+                    )
+                    collection.remove(eo_object)
+
+                else:
+                    self.print_wrn(
+                        "Collection %s does not contain %s" 
+                        % (collection, eo_object)
+                    )
+
+        except Exception as e:
+            self.print_traceback(e, kwargs)
+            raise CommandError("Unlinking failed: %s" % (e))

eoxserver/resources/coverages/management/commands/eoxs_dataset_register.py

@@ -156,6 +156,12 @@ class Command(CommandOutputMixIn, BaseCommand):
             help=("Optional. If the coverage with the given identifier already "
                   "exists, replace it. Without this flag, this would result in "
                   "an error.")
+        ),
+
+        make_option("--scheme",
+            action="store", default="GDAL",
+            help=("Optional. How the input files shall be treated and "
+                  "registered. Default is the 'GDAL' scheme.")
         )
     )
 

eoxserver/resources/coverages/metadata/formats/native_config.py

@@ -0,0 +1,76 @@
+#-------------------------------------------------------------------------------
+#
+# Project: EOxServer <http://eoxserver.org>
+# Authors: Fabian Schindler <fabian.schindler@eox.at>
+#
+#-------------------------------------------------------------------------------
+# Copyright (C) 2014 EOX IT Services GmbH
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#-------------------------------------------------------------------------------
+
+import os.path
+from cStringIO import StringIO
+from ConfigParser import RawConfigParser
+
+from eoxserver.core import Component, implements
+from eoxserver.core.decoders import config
+from eoxserver.resources.coverages.metadata.interfaces import (
+    MetadataReaderInterface
+)
+
+
+class NativeConfigFormatReader(Component):
+    implements(MetadataReaderInterface)
+
+    def open_reader(self, obj):
+        if isinstance(obj, basestring):
+            try:
+                parser = RawConfigParser()
+                if os.path.exists(obj):
+                    parser.read((obj,))
+                else:
+                    parser.readfp(StringIO(obj))
+
+                return NativeConfigReader(parser)
+            except:
+                pass
+        return None
+
+    def test(self, obj):
+        try:
+            reader = self.open_reader(obj)
+            reader.range_type_name
+            return True
+        except:
+            return False
+
+    def get_format_name(self, obj):
+        return "native_config"
+
+    def read(self, obj):
+        reader = self.open_reader(obj)
+        if reader:
+            return {
+                "range_type_name": reader.range_type_name
+            }
+
+
+class NativeConfigReader(config.Reader):
+    range_type_name = config.Option(section="range_type")

eoxserver/resources/coverages/models.py

@@ -157,8 +157,8 @@ class Meta:
 
 
 class DataSource(backends.Dataset):
-    pattern = models.CharField(max_length=32, null=False, blank=False)
-    collection = models.ForeignKey("Collection")
+    pattern = models.CharField(max_length=512, null=False, blank=False)
+    collection = models.ForeignKey("Collection", related_name="data_sources")
 
 
 #===============================================================================