Merge pull request #557 from EOxServer/storage-streaming

Storage streaming and RGBAlpha browse support and antimeridian fix

documentation/users/backends.rst

@@ -38,7 +38,7 @@ and other files that either reside on a local or remote storage.
 
 The backends have a static representation in the database (i.e the
 data models) and a dynamic behavioral implementation: the handlers.
-The combintation of both allows the registration of storages,
+The combination of both allows the registration of storages,
 backend authorization and data items and the access at runtime.
 
 
@@ -83,6 +83,9 @@ for a ZIP file storage the URL is the path to the ZIP file.
 
 Each storage can be given a name, which helps with management.
 
+Storage can have `streaming` property set to `true` which wherever possible uses
+respective `streaming` version of `/vsi` file accessor, like `/vsicurl_streaming`.
+
 A Storage can be linked to a `Storage Auth`_ model, which allows
 to specify authorization credentials.
 

eoxserver/backends/access.py

@@ -101,7 +101,7 @@ def retrieve(data_item, cache=None):
             tmp_path = cache.relative_path(item_id)
             if not cache.contains(item_id):
                 # actually retrieve the item when not in the cache
-                handler = handler_cls(path or storage.url)
+                handler = handler_cls(path or storage.url, storage.streaming)
                 use_cache, path = handler.retrieve(
                     path or child_storage.url, tmp_path
                 )
@@ -112,7 +112,7 @@ def retrieve(data_item, cache=None):
 
         if storage_handlers:
             storage, handler_cls = storage_handlers[-1]
-            handler = handler_cls(path)
+            handler = handler_cls(path, storage.streaming)
             return handler.retrieve(data_item.location)[1]
 
 
@@ -159,7 +159,7 @@ def get_vsi_storage_path(storage, location=None):
     while storage:
         handler_cls = get_handler_class_for_model(storage)
         if handler_cls:
-            handler = handler_cls(storage.url)
+            handler = handler_cls(storage.url, storage.streaming)
             location = handler.get_vsi_path(location or '')
         else:
             raise AccessError(
@@ -178,7 +178,7 @@ def get_vsi_env(storage):
     while storage:
         handler_cls = get_handler_class_for_model(storage)
         if handler_cls:
-            handler = handler_cls(storage.url)
+            handler = handler_cls(storage.url, storage.streaming)
             env.update(handler.get_vsi_env())
         else:
             raise AccessError(

eoxserver/backends/management/commands/storage.py

@@ -72,6 +72,12 @@ def add_arguments(self, parser):
             dest='storage_auth_name', default=None,
             help='The name of the storage auth to use. Optional',
         )
+        create_parser.add_argument(
+            '--streaming', action="store_true",
+            default=False,
+            help="""If used, respective streaming version of /vsi file 
+            accessor will be used."""
+        )
 
         for parser in [list_parser, env_parser]:
             parser.add_argument(
@@ -103,7 +109,7 @@ def handle(self, subcommand, name, *args, **kwargs):
             self.handle_env(name, *args, **kwargs)
 
     def handle_create(self, name, url, type_name, parent_name,
-                      storage_auth_name, **kwargs):
+                      storage_auth_name, streaming, **kwargs):
         """ Handle the creation of a new storage.
         """
         url = url[0]
@@ -141,7 +147,7 @@ def handle_create(self, name, url, type_name, parent_name,
 
         backends.Storage.objects.create(
             name=name, url=url, storage_type=type_name, parent=parent,
-            storage_auth=storage_auth,
+            storage_auth=storage_auth, streaming=streaming,
         )
 
         self.print_msg(

eoxserver/backends/migrations/0004_storage_streaming.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.18 on 2023-04-07 07:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('backends', '0003_nameblank'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='storage',
+            name='streaming',
+            field=models.BooleanField(blank=True, null=True),
+        ),
+    ]

eoxserver/backends/models.py

@@ -65,6 +65,7 @@ class Storage(models.Model):
     storage_type = models.CharField(max_length=32, **mandatory)
     name = models.CharField(max_length=1024, null=True, blank=True, unique=True)
     storage_auth = models.ForeignKey(StorageAuth, on_delete=models.CASCADE, **optional)
+    streaming = models.BooleanField(null=True, blank=True)
 
     parent = models.ForeignKey("self", on_delete=models.CASCADE, **optional)
 

eoxserver/backends/storages.py

@@ -110,9 +110,10 @@ class ZIPStorageHandler(BaseStorageHandler):
 
     is_local = True
 
-    def __init__(self, package_filename):
+    def __init__(self, package_filename, streaming):
         self.package_filename = package_filename
         self.zipfile = None
+        self.streaming = streaming
 
     def __enter__(self):
         self.zipfile = zipfile.ZipFile(self.package_filename, "r")
@@ -153,9 +154,10 @@ class TARStorageHandler(BaseStorageHandler):
 
     is_local = True
 
-    def __init__(self, package_filename):
+    def __init__(self, package_filename, streaming):
         self.package_filename = package_filename
         self.tarfile = None
+        self.streaming = streaming
 
     def __enter__(self):
         self.tarfile = tarfile.TarFile(self.package_filename, "r")
@@ -197,8 +199,9 @@ class DirectoryStorageHandler(BaseStorageHandler):
 
     is_local = True
 
-    def __init__(self, dirpath):
+    def __init__(self, dirpath, streaming):
         self.dirpath = dirpath
+        self.streaming = streaming
 
     def retrieve(self, location, path):
         return False, os.path.join(self.dirpath, location)
@@ -224,15 +227,20 @@ class HTTPStorageHandler(BaseStorageHandler):
     allows_child_storages = True
     allows_parent_storage = False
 
-    def __init__(self, url):
+    def __init__(self, url, streaming):
         self.url = url
+        self.streaming = streaming
 
     def retrieve(self, location, path):
         request.urlretrieve(parse.urljoin(self.url, location), path)
         return True, path
 
     def get_vsi_path(self, location):
-        return '/vsicurl/%s' % parse.urljoin(self.url, location)
+        if self.streaming:
+            prefix = '/vsicurl_streaming/'
+        else:
+            prefix = '/vsicurl/'
+        return '%s%s' % (prefix, parse.urljoin(self.url, location))
 
     @classmethod
     def test(cls, locator):
@@ -251,10 +259,11 @@ class FTPStorageHandler(BaseStorageHandler):
     allows_parent_storage = True
     allows_parent_storage = False
 
-    def __init__(self, url):
+    def __init__(self, url, streaming):
         self.url = url
         self.parsed_url = urlparse(url)
         self.ftp = None
+        self.streaming=streaming
 
     def __enter__(self):
         self.ftp = ftplib.FTP()
@@ -285,7 +294,11 @@ def list_files(self, location, glob_pattern=None):
         return filenames
 
     def get_vsi_path(self, location):
-        return '/vsicurl/%s' % parse.urljoin(self.url, location)
+        if self.streaming:
+            prefix = '/vsicurl_streaming/'
+        else:
+            prefix = '/vsicurl/'
+        return '%s%s' % (prefix, parse.urljoin(self.url, location))
 
     @classmethod
     def test(cls, locator):
@@ -301,8 +314,9 @@ class SwiftStorageHandler(BaseStorageHandler):
     allows_parent_storage = False
     allows_child_storages = True
 
-    def __init__(self, url):
+    def __init__(self, url, streaming):
         self.container = url
+        self.streaming = streaming
 
     def retrieve(self, location, path):
         pass
@@ -311,7 +325,12 @@ def list_files(self, location, glob_pattern=None):
         return []
 
     def get_vsi_path(self, location):
-        return vsi.join('/vsiswift/%s' % self.container, location)
+        if self.streaming:
+            prefix = '/vsiswift_streaming'
+        else:
+            prefix = '/vsiswift'
+        base_path = f'{prefix}/{self.container}' if self.container else prefix
+        return vsi.join(base_path, location)
 
     @classmethod
     def test(cls, locator):
@@ -324,8 +343,9 @@ class S3StorageHandler(BaseStorageHandler):
     allows_parent_storage = False
     allows_child_storages = True
 
-    def __init__(self, url):
+    def __init__(self, url, streaming):
         self.bucket = url
+        self.streaming = streaming
 
     def retrieve(self, location, path):
         pass
@@ -334,13 +354,11 @@ def list_files(self, location, glob_pattern=None):
         return []
 
     def get_vsi_path(self, location):
-        import logging
-        logger = logging.getLogger(__name__)
-
-        # logger.debug()
-
-
-        base_path = '/vsis3/%s' % self.bucket if self.bucket else '/vsis3'
+        if self.streaming:
+            prefix = '/vsis3_streaming'
+        else:
+            prefix = '/vsis3'
+        base_path = f'{prefix}/{self.bucket}' if self.bucket else prefix
         return vsi.join(base_path, location)
 
     @classmethod

eoxserver/contrib/vrt.py

@@ -498,6 +498,9 @@ def stack_bands(filenames, env, save=None):
 
     return out_ds
 
+def sign_abs(x):
+    return 0.0 if abs(x) == 0 else x / abs(x)
+
 def with_extent(filename, extent, save=None):
     """ Create a VRT and override the underlying files geolocation
     """
@@ -509,6 +512,9 @@ def with_extent(filename, extent, save=None):
     x = extent[0]
     y = extent[3]
 
+    source_geotransform = src_ds.GetGeoTransform()
+    resy_sign_north_up = sign_abs(source_geotransform[5])
+
     resx = abs(extent[2] - extent[0]) / width
     resy = abs(extent[3] - extent[1]) / height
     out_ds.SetGeoTransform([
@@ -517,6 +523,6 @@ def with_extent(filename, extent, save=None):
         0,
         y,
         0,
-        resy,
+        resy_sign_north_up * resy,
     ])
     return out_ds

eoxserver/render/browse/objects.py

@@ -29,6 +29,7 @@
 
 from django.contrib.gis.geos import Polygon
 from django.contrib.gis.gdal import SpatialReference, CoordTransform, DataSource
+from django.conf import settings
 
 from eoxserver.contrib import gdal
 from eoxserver.backends.access import get_vsi_path, get_vsi_env, gdal_open
@@ -38,6 +39,7 @@
 BROWSE_MODE_RGB = "rgb"
 BROWSE_MODE_RGBA = "rgba"
 BROWSE_MODE_GRAYSCALE = "grayscale"
+DEFAULT_EOXS_LAYER_SUFFIX_SEPARATOR = '__'
 
 
 OptionalNumeric = Optional[Union[float, int]]
@@ -111,10 +113,13 @@ def from_model(cls, product_model, browse_model):
         ds = gdal_open(browse_model)
         mode = _get_ds_mode(ds)
         ds = None
-
-        if browse_model.browse_type:
-            name = '%s__%s' % (
-                product_model.identifier, browse_model.browse_type.name
+        suffix_separator = getattr(
+            settings, 'EOXS_LAYER_SUFFIX_SEPARATOR',
+            DEFAULT_EOXS_LAYER_SUFFIX_SEPARATOR
+        )
+        if browse_model.browse_type and browse_model.browse_type.name:
+            name = '%s%s%s' % (
+                product_model.identifier, suffix_separator, browse_model.browse_type.name
             )
         else:
             name = product_model.identifier

eoxserver/render/mapserver/factories.py

@@ -40,7 +40,7 @@
 from eoxserver.contrib import mapserver as ms
 from eoxserver.contrib import vsi, vrt, gdal, osr
 from eoxserver.render.browse.objects import (
-    Browse, GeneratedBrowse, BROWSE_MODE_GRAYSCALE
+    Browse, GeneratedBrowse, BROWSE_MODE_GRAYSCALE, BROWSE_MODE_RGBA
 )
 from eoxserver.render.browse.generate import (
     generate_browse, FilenameGenerator
@@ -183,7 +183,7 @@ def create_coverage_layer(self, map_obj: Map, coverage: Coverage, fields: List[F
             sr = osr.SpatialReference(map_obj.getProjection())
 
         layer_objs = _create_raster_layer_objs(
-            map_obj, extent, sr, data, filename_generator
+            map_obj, extent, sr, data, filename_generator, location.env,
         )
 
         for i, layer_obj in enumerate(layer_objs):
@@ -355,11 +355,10 @@ def make_browse_layer_generator(self, map_obj, browses, map_,
                     )
                 layer_objs = _create_raster_layer_objs(
                     map_obj, browse.extent, browse.spatial_reference,
-                    creation_info.filename, filename_generator
+                    creation_info.filename, filename_generator, creation_info.env, browse.mode,
                 )
 
                 for layer_obj in layer_objs:
-                    layer_obj.data = creation_info.filename
                     if creation_info.env:
                         ms.set_env(map_obj, creation_info.env, True)
 
@@ -447,10 +446,8 @@ def make_browse_layer_generator(self, map_obj, browses, map_,
             elif isinstance(browse, Browse):
                 layer_objs = _create_raster_layer_objs(
                     map_obj, browse.extent, browse.spatial_reference,
-                    browse.filename, filename_generator
+                    browse.filename, filename_generator, browse.env, browse.mode,
                 )
-                for layer_obj in layer_objs:
-                    layer_obj.data = browse.filename
                 ms.set_env(map_obj, browse.env, True)
             elif browse is None:
                 # TODO: figure out why and deal with it?
@@ -661,15 +658,16 @@ def create(self, map_obj, layer):
 # ------------------------------------------------------------------------------
 
 
-def _create_raster_layer_objs(map_obj, extent, sr, data, filename_generator,
-                              resample=None) -> List[ms.layerObj]:
+def _create_raster_layer_objs(map_obj, extent, sr, data, filename_generator, env,
+                              browse_mode=None, resample=None) -> List[ms.layerObj]:
     layer_obj = ms.layerObj(map_obj)
     layer_obj.type = ms.MS_LAYER_RASTER
     layer_obj.status = ms.MS_ON
 
     layer_obj.data = data
-
-    layer_obj.offsite = ms.colorObj(0, 0, 0)
+    # assumption that RGBA already has transparency in alpha band
+    if browse_mode != BROWSE_MODE_RGBA:
+        layer_obj.offsite = ms.colorObj(0, 0, 0)
 
     if extent:
         layer_obj.setMetaData("wms_extent", "%f %f %f %f" % extent)
@@ -693,10 +691,12 @@ def _create_raster_layer_objs(map_obj, extent, sr, data, filename_generator,
         wrapped_layer_obj.status = ms.MS_ON
 
         wrapped_data = filename_generator.generate()
-        vrt.with_extent(data, wrapped_extent, wrapped_data)
+        with gdal.config_env(env):
+            vrt.with_extent(data, wrapped_extent, wrapped_data)
         wrapped_layer_obj.data = wrapped_data
-
-        wrapped_layer_obj.offsite = ms.colorObj(0, 0, 0)
+        # assumption that RGBA already has transparency in alpha band
+        if browse_mode != BROWSE_MODE_RGBA:
+            wrapped_layer_obj.offsite = ms.colorObj(0, 0, 0)
 
         wrapped_layer_obj.setMetaData("ows_srs", short_epsg)
         wrapped_layer_obj.setMetaData("wms_srs", short_epsg)