Merge pull request #1614 from dassaniansh/addingExtent

Adding extent

.github/workflows/python-package.yml

@@ -57,10 +57,16 @@ jobs:
           TZ: America/New_York
           DEBIAN_FRONTEND: noninteractive
         run: |
+          sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
           sudo apt-get update && sudo apt-get install -y --no-install-recommends apt-utils
           sudo apt-get install -y --force-yes tzdata
           sudo apt-get install -y --force-yes build-essential wget git locales locales-all > /dev/null
           sudo apt-get install -y --force-yes libpq-dev
+          sudo apt-get install libgdal-dev
+          sudo apt-get install gdal-bin
+          export CPLUS_INCLUDE_PATH=/usr/include/gdal
+          export C_INCLUDE_PATH=/usr/include/gdal
+          pip install GDAL==3.2.3 
           sudo apt-get install -y --force-yes postgis
           
       - name: Setup paths and files

appveyor.yml

@@ -1,9 +1,7 @@
 environment:
-
   matrix:
-
-    - PYTHON: "C:\\Python36-x64"
-      PYTHON_VERSION: "3.6.8"
+    - PYTHON: "C:\\Python38-x64"
+      PYTHON_VERSION: "3.8.x"
       PYTHON_ARCH: "64"
 
   # postgres
@@ -29,19 +27,22 @@ services:
 
 init:
   - SET PATH=%POSTGRES_PATH%\bin;%MYSQL_PATH%\bin;%PATH%
-
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
 install:
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "SET PYTHONPATH=%PYTHON%\\Lib\\site-packages;%PYTHONPATH%"
   - "%PYTHON%\\python.exe -m pip install --upgrade pip"
   - "%PYTHON%\\python.exe -m pip install wheel"
   - "%PYTHON%\\python.exe -m pip install -r requirements.txt"
+  - "%PYTHON%\\python.exe -m pip install pymyinstall"
   - "%PYTHON%\\python.exe -m pip install codecov"
   - "%PYTHON%\\python.exe -m pip install nose-cov"
   - "%PYTHON%\\python.exe -m pip install -U pytest"
   - "%PYTHON%\\python.exe -m pip install -U flake8"
   - "%PYTHON%\\python.exe -m pip install -U yapf"
   - "%PYTHON%\\python.exe -m pip install -U pylint"
-
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "%PYTHON%\\Scripts\\pymy_install3 cartopy pyproj shapely fiona gdal llvmlite numba cpyquickhelper"
+  - set PYTHONPATH=src
   - "%PYTHON%\\python.exe -m  pip --version"
   - "python --version"
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""

docker/Dockerfile

@@ -3,14 +3,23 @@ FROM ubuntu:18.04
 
 MAINTAINER Weecology "https://github.com/weecology/retriever"
 
-RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
-
-# Manually install tzdata to allow for non-interactive install
-RUN apt-get install -y --force-yes tzdata
-
-RUN apt-get install -y --force-yes build-essential wget git locales locales-all > /dev/null &&\
+# Installing Gdal and configuring it
+RUN add-apt-repository ppa:ubuntugis/ppa &&\
+    apt-get update && apt-get install -y --no-install-recommends apt-utils &&\
+    # Manually install tzdata to allow for non-interactive install
+    apt-get install -y --force-yes tzdata &&\
+    apt-get install -y --force-yes build-essential wget git locales locales-all > /dev/null &&\
     apt-get install -y --force-yes postgresql-client mariadb-client > /dev/null &&\
-    apt-get install -y --force-yes libpq-dev
+    apt-get install -y --force-yes libpq-dev &&\
+    apt-get install gdal-bin &&\
+    apt-get install libgdal-dev &&\
+    export CPLUS_INCLUDE_PATH=/usr/include/gdal &&\
+    export C_INCLUDE_PATH=/usr/include/gdal &&\
+    pip install GDAL 
+
+# Installing GDAL
+RUN apt-add-repository ppa:ubuntugis/ubuntugis-unstable &&\
+    apt install -y gdal-bin python-gdal python3-gdal
 
 # Set encoding
 ENV LC_ALL en_US.UTF-8

retriever/engines/postgres.py

@@ -1,7 +1,7 @@
 import os
 import sys
 import subprocess
-
+from osgeo import gdal
 from retriever.lib.models import Engine
 
 
@@ -165,7 +165,74 @@ def insert_raster(self, path=None, srid=4326):
 
         if not path:
             path = Engine.format_data_dir(self)
-
+        is_cli_extent = hasattr(self, 'opts') and self.opts["bbox"]
+        is_resource_extent = hasattr(self.script.tables[self.table.name], 'extent')
+        is_global_script_extent = hasattr(self.script, 'extent')
+        if any([is_cli_extent, is_resource_extent, is_global_script_extent]):
+            # bounding box array: bbox[xmin, ymax, xmax, ymin]
+            bbox = []
+
+            if self.opts["bbox"]:
+                bbox.append(self.opts["bbox"][0])
+                bbox.append(self.opts["bbox"][1])
+                bbox.append(self.opts["bbox"][2])
+                bbox.append(self.opts["bbox"][3])
+
+            else:
+                if is_global_script_extent and self.script.extent:
+                    bbox.append(self.script.extent["xMin"])
+                    bbox.append(self.script.extent["yMax"])
+                    bbox.append(self.script.extent["xMax"])
+                    bbox.append(self.script.extent["yMin"])
+
+                else:
+                    if is_resource_extent and self.script.tables[self.table.name].extent:
+                        bbox.append(self.script.tables[self.table.name].extent["xMin"])
+                        bbox.append(self.script.tables[self.table.name].extent["yMax"])
+                        bbox.append(self.script.tables[self.table.name].extent["xMax"])
+                        bbox.append(self.script.tables[self.table.name].extent["yMin"])
+
+            bbox = [int(i) for i in bbox]
+
+            if bbox and bbox[2] > bbox[0] and bbox[1] > bbox[3]:
+                if self.script.tables[self.table.name].extensions:
+                    converted_tif = path[:-3] + "tif"
+                    if self.script.tables[self.table.name].extensions[0] == "bil":
+                        conversion = "gdal_translate -co 'COMPRESS=LZW' {path} {converted_tif}".format(
+                            path=os.path.normpath(path), converted_tif=converted_tif)
+                        os.system(conversion)
+
+                    ds = gdal.Open(converted_tif)
+
+                    info = gdal.Info(converted_tif, format='json')
+                    coordinates = info['wgs84Extent']['coordinates'][0]
+
+                    if bbox[0] < coordinates[2][0] and bbox[2] > coordinates[0][
+                            0] and bbox[1] > coordinates[1][1] and bbox[3] < coordinates[
+                                0][1]:
+
+                        if bbox[0] < coordinates[0][0]:
+                            bbox[0] = coordinates[0][0]
+
+                        if bbox[1] > coordinates[0][1]:
+                            bbox[1] = coordinates[0][1]
+
+                        if bbox[2] > coordinates[2][0]:
+                            bbox[2] = coordinates[2][0]
+
+                        if bbox[3] < coordinates[2][1]:
+                            bbox[3] = coordinates[2][1]
+
+                        i = converted_tif.find(".tif")
+                        location_of_cropped_tif = converted_tif[:i] + \
+                            "crop" + converted_tif[i:]
+                        ds = gdal.Translate(location_of_cropped_tif, ds, projWin=bbox)
+                        ds = None
+                        path = location_of_cropped_tif
+                    else:
+                        print("Bounding Box exceds image boundaries")
+                elif bbox:
+                    print("Invalid value of Extent, bbox[xmin, ymax, xmax, ymin]")
         raster_sql = ('raster2pgsql -Y -M -d -I -l 2 -s {SRID} "{path}"'
                       " -F -t 100x100 {SCHEMA_DBTABLE}".format(
                           SRID=srid,

retriever/lib/create_scripts.py

@@ -113,7 +113,12 @@ def __init__(self, **kwargs):
         self.driver_name = "ESRI Shapefile"
         self.spatial_ref = "spatial_ref"
         self.resources = []
-        self.extent = ""
+        self.extent = {
+            "xMin": "fill a numerical value",
+            "yMax": "fill numerical value",
+            "xMax": "fill numerical value",
+            "yMin": "fill numerical value"
+        }
         self.geom_type = ""
 
     def get_source(self, source, driver_name=None):
@@ -186,6 +191,7 @@ def __init__(self, **kwargs):
         self.group_count = "--Number of groups in the dataset if applicable"
         self.dataset_count = "--The number of individual datasets"
         self.transform = ""
+        self.extent = []
         self.resources = []
 
     def get_source(self, file_path, driver=None):

retriever/lib/get_opts.py

@@ -117,7 +117,7 @@
 download_parser.add_argument('-b',
                              '--bbox',
                              nargs=4,
-                             help='Set bounding box xmin, ymin, xmax, ymax',
+                             help='Set bounding box xmin, ymax, xmax, ymin',
                              required=False)
 
 ls_parser.add_argument('-l', help='search datasets with specific license(s)',

test/raw_data_gis/scripts/tests_bioclim_bio1.json

@@ -0,0 +1,54 @@
+{
+    "name": "bioclim-bio1",
+    "url": "http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/bio_2-5m_bil.zip",
+    "title": "Bioclim 2.5 Minute Climate Data",
+    "ref":"http://worldclim.org/bioclim",
+    "driver": "EHdr",
+    "archived": "zip",
+    "extract_all": "True",
+    "description": "Bioclimatic variables that are derived from the monthly temperature and rainfall values in order to generate more biologically meaningful variables.",
+    "licenses": [{"name": "CC-BY-SA"}],
+    "keywords": [
+        "climate",
+        "spatial-data",
+        "global"
+    ],
+    "citation": "Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. Jones and A. Jarvis, 2005. Very high resolution interpolated climate surfaces for global land areas. International Journal of Climatology 25: 1965-1978.",
+    "version": "1.3.0",
+    "homepage": "http://worldclim.org/bioclim",
+    "colums": 8640,
+    "rows": 3600,
+    "datum": "--Coordinate Reference System",
+    "projection": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]]",
+    "file_size": "--size of file on disk",
+    "group_count": "--Number of groups in the dataset if applicable",
+    "dataset_count": "--The number of individual datasets",
+    "transform": {
+        "xOrigin": -180.00000000000335,
+        "pixelWidth": 0.041666666666667,
+        "rotation_2": 0.0,
+        "yOrigin": 90.00000000000003,
+        "rotation_4": 0.0,
+        "pixelHeight": -0.041666666666667
+    },
+    "resources": [
+        {
+            "extensions": ["bil", "hdr"],
+            "format": "raster",
+            "name": "bio1",
+            "path":"bio1.bil",
+            "other_paths": "bio1.hdr",
+            "no_data_value": -9999.0,
+            "min": -272.0,
+            "max": 312.0,
+            "scale": 1.0,
+            "color_table": null,
+            "statistics": {
+                "minimum": -272.0,
+                "maximum": 312.0,
+                "mean": 81.1423446407,
+                "stddev": 148.2358307206
+            }
+        }
+    ]
+}

test/raw_data_gis/scripts/tests_bioclim_bio1_crop_extent.json

@@ -0,0 +1,60 @@
+{
+    "name": "bioclim-bio1-crop-extent",
+    "url": "http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/bio_2-5m_bil.zip",
+    "title": "Bioclim 2.5 Minute Climate Data",
+    "ref":"http://worldclim.org/bioclim",
+    "driver": "EHdr",
+    "archived": "zip",
+    "extract_all": "True",
+    "description": "Bioclimatic variables that are derived from the monthly temperature and rainfall values in order to generate more biologically meaningful variables.",
+    "licenses": [{"name": "CC-BY-SA"}],
+    "keywords": [
+        "climate",
+        "spatial-data",
+        "global"
+    ],
+    "citation": "Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. Jones and A. Jarvis, 2005. Very high resolution interpolated climate surfaces for global land areas. International Journal of Climatology 25: 1965-1978.",
+    "version": "1.3.0",
+    "homepage": "http://worldclim.org/bioclim",
+    "colums": 8640,
+    "rows": 3600,
+    "datum": "--Coordinate Reference System",
+    "projection": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]]",
+    "file_size": "--size of file on disk",
+    "group_count": "--Number of groups in the dataset if applicable",
+    "dataset_count": "--The number of individual datasets",
+    "transform": {
+        "xOrigin": -180.00000000000335,
+        "pixelWidth": 0.041666666666667,
+        "rotation_2": 0.0,
+        "yOrigin": 90.00000000000003,
+        "rotation_4": 0.0,
+        "pixelHeight": -0.041666666666667
+    },
+    "extent": {
+        "xMin": 60,
+        "yMax": 50,
+        "xMax": 100,
+        "yMin": 0
+      },
+    "resources": [
+        {
+            "extensions": ["bil", "hdr"],
+            "format": "raster",
+            "name": "bio1",
+            "path":"bio1.bil",
+            "other_paths": "bio1.hdr",
+            "no_data_value": -9999.0,
+            "min": -272.0,
+            "max": 312.0,
+            "scale": 1.0,
+            "color_table": null,
+            "statistics": {
+                "minimum": -272.0,
+                "maximum": 312.0,
+                "mean": 81.1423446407,
+                "stddev": 148.2358307206
+            }
+        }
+    ]
+}

test/raw_data_gis/scripts/tests_bioclim_bio1_crop_extent2.json

@@ -0,0 +1,61 @@
+{
+    "name": "bioclim-bio1-crop-extent2",
+    "url": "http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/bio_2-5m_bil.zip",
+    "title": "Bioclim 2.5 Minute Climate Data",
+    "ref":"http://worldclim.org/bioclim",
+    "driver": "EHdr",
+    "archived": "zip",
+    "extract_all": "True",
+    "description": "Bioclimatic variables that are derived from the monthly temperature and rainfall values in order to generate more biologically meaningful variables.",
+    "licenses": [{"name": "CC-BY-SA"}],
+    "keywords": [
+        "climate",
+        "spatial-data",
+        "global"
+    ],
+    "citation": "Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. Jones and A. Jarvis, 2005. Very high resolution interpolated climate surfaces for global land areas. International Journal of Climatology 25: 1965-1978.",
+    "version": "1.3.0",
+    "homepage": "http://worldclim.org/bioclim",
+    "colums": 8640,
+    "rows": 3600,
+    "datum": "--Coordinate Reference System",
+    "projection": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]]",
+    "file_size": "--size of file on disk",
+    "group_count": "--Number of groups in the dataset if applicable",
+    "dataset_count": "--The number of individual datasets",
+    "transform": {
+        "xOrigin": -180.00000000000335,
+        "pixelWidth": 0.041666666666667,
+        "rotation_2": 0.0,
+        "yOrigin": 90.00000000000003,
+        "rotation_4": 0.0,
+        "pixelHeight": -0.041666666666667
+    },
+
+    "resources": [
+        {
+            "extensions": ["bil", "hdr"],
+            "format": "raster",
+            "name": "bio1",
+            "path":"bio1.bil",
+            "other_paths": "bio1.hdr",
+            "no_data_value": -9999.0,
+            "min": -272.0,
+            "max": 312.0,
+            "scale": 1.0,
+            "color_table": null,
+            "statistics": {
+                "minimum": -272.0,
+                "maximum": 312.0,
+                "mean": 81.1423446407,
+                "stddev": 148.2358307206
+            },
+            "extent": {
+                "xMin": 60,
+                "yMax": 50,
+                "xMax": 100,
+                "yMin": 0
+              }
+        }
+    ]
+}