Merge pull request #110 from DHI-GRAS/masked

Bug fixes

setup.py

@@ -24,10 +24,10 @@
         'flask_cors',
         'marshmallow>=3.0.0b12',
         'mercantile',
-        'numpy',
+        'numpy>=1.15',
         'pillow',
         'shapely',
-        'rasterio>=1.0,<1.0.7',
+        'rasterio>=1.0',
         'shapely',
         'toml',
         'tqdm'

terracotta/config.py

@@ -33,7 +33,6 @@ class TerracottaSettings(NamedTuple):
 
     #: Size of raster file in-memory cache in bytes
     RASTER_CACHE_SIZE: int = 1024 * 1024 * 490  # 490 MB
-    METADATA_CACHE_SIZE: int = 1024 * 1024 * 10  # 10 MB
 
     #: Tile size to return if not given in parameters
     DEFAULT_TILE_SIZE: Tuple[int, int] = (256, 256)
@@ -81,7 +80,6 @@ class SettingSchema(Schema):
     )
 
     RASTER_CACHE_SIZE = fields.Integer()
-    METADATA_CACHE_SIZE = fields.Integer()
 
     DEFAULT_TILE_SIZE = fields.List(fields.Integer(), validate=validate.Length(equal=2))
     LAZY_LOADING_MAX_SHAPE = fields.List(fields.Integer(), validate=validate.Length(equal=2))

terracotta/drivers/base.py

@@ -67,7 +67,6 @@ def get_metadata(self, keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[st
           - range: global minimum and maximum value in dataset
           - bounds: physical bounds covered by dataset
           - convex_hull: GeoJSON shape specifying total data coverage
-          - nodata: data value denoting missing or invalid data
           - percentiles: array of pre-computed percentiles in range(1, 100)
           - mean: global mean
           - stdev: global standard deviation
@@ -80,7 +79,6 @@ def get_metadata(self, keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[st
     def get_raster_tile(self, keys: Union[Sequence[str], Mapping[str, str]], *,
                         bounds: Sequence[float] = None,
                         tile_size: Sequence[int] = (256, 256),
-                        nodata: Number = 0,
                         preserve_values: bool = False,
                         asynchronous: bool = False) -> Any:
         """Get raster tile as a NumPy array for given keys and bounds.

terracotta/drivers/mysql.py

@@ -65,7 +65,6 @@ class MySQLDriver(RasterDriver):
         ('bounds_south', 'REAL'),
         ('bounds_west', 'REAL'),
         ('convex_hull', 'LONGTEXT'),
-        ('nodata', 'REAL'),
         ('valid_percentage', 'REAL'),
         ('min', 'REAL'),
         ('max', 'REAL'),
@@ -277,31 +276,35 @@ def _get_keys(self) -> OrderedDict:
     @requires_connection
     @convert_exceptions('Could not retrieve datasets')
     def get_datasets(self, where: Mapping[str, str] = None,
-                     page: int = 0, limit: int = 100) -> Dict[Tuple[str, ...], str]:
+                     page: int = 0, limit: int = None) -> Dict[Tuple[str, ...], str]:
         """Retrieve keys of datasets matching given pattern"""
         cursor = self._cursor
 
-        # explicitly cast to int to prevent SQL injection
-        page_query = f'LIMIT {int(limit)} OFFSET {int(page) * int(limit)}'
+        if limit is not None:
+            # explicitly cast to int to prevent SQL injection
+            page_fragment = f'LIMIT {int(limit)} OFFSET {int(page) * int(limit)}'
+        else:
+            page_fragment = ''
+
+        # sort by keys to ensure deterministic results
+        order_fragment = f'ORDER BY {", ".join(self.key_names)}'
 
         if where is None:
-            cursor.execute(f'SELECT * FROM datasets {page_query}')
-            rows = cursor.fetchall()
+            cursor.execute(f'SELECT * FROM datasets {order_fragment} {page_fragment}')
         else:
             if not all(key in self.key_names for key in where.keys()):
                 raise exceptions.InvalidKeyError('Encountered unrecognized keys in '
                                                  'where clause')
-            where_string = ' AND '.join([f'{key}=%s' for key in where.keys()])
-            cursor.execute(f'SELECT * FROM datasets WHERE {where_string}', list(where.values()))
-            rows = cursor.fetchall()
-
-        if rows is None:
-            rows = tuple()
+            where_fragment = ' AND '.join([f'{key}=%s' for key in where.keys()])
+            cursor.execute(
+                f'SELECT * FROM datasets WHERE {where_fragment} {order_fragment} {page_fragment}',
+                list(where.values())
+            )
 
         def keytuple(row: Dict[str, Any]) -> Tuple[str, ...]:
             return tuple(row[key] for key in self.key_names)
 
-        return {keytuple(row): row['filepath'] for row in rows}
+        return {keytuple(row): row['filepath'] for row in cursor}
 
     @staticmethod
     def _encode_data(decoded: Mapping[str, Any]) -> Dict[str, Any]:
@@ -312,7 +315,6 @@ def _encode_data(decoded: Mapping[str, Any]) -> Dict[str, Any]:
             'bounds_south': decoded['bounds'][2],
             'bounds_west': decoded['bounds'][3],
             'convex_hull': json.dumps(decoded['convex_hull']),
-            'nodata': decoded['nodata'],
             'valid_percentage': decoded['valid_percentage'],
             'min': decoded['range'][0],
             'max': decoded['range'][1],
@@ -329,7 +331,6 @@ def _decode_data(encoded: Mapping[str, Any]) -> Dict[str, Any]:
         decoded = {
             'bounds': tuple([encoded[f'bounds_{d}'] for d in ('north', 'east', 'south', 'west')]),
             'convex_hull': json.loads(encoded['convex_hull']),
-            'nodata': encoded['nodata'],
             'valid_percentage': encoded['valid_percentage'],
             'range': (encoded['min'], encoded['max']),
             'mean': encoded['mean'],