Merge 2ee5186 into 774af92

.docker/.datacube_integration.conf

@@ -0,0 +1,5 @@
+[datacube]
+db_hostname: postgres
+db_database: opendatacube_test
+db_username: opendatacube
+db_password: opendatacubepassword

.docker/create_db.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+PGPASSWORD=${DB_PASSWORD} psql -h ${DB_HOSTNAME} -U ${DB_USERNAME} -c 'create database opendatacube_test'

Dockerfile

@@ -15,7 +15,6 @@ RUN pip3 install --upgrade pip \
     && rm -rf $HOME/.cache/pip
 
 WORKDIR /code
-RUN mkdir /code/product-summaries
 
 ADD . .
 

Dockerfile.Dev

@@ -1,35 +1,41 @@
-FROM opendatacube/datacube-core:latest
+FROM ubuntu:bionic
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y \
-    python3-fiona \
-    python3-shapely \
-    libpng-dev \
-    wget \
-    vim \
-    unzip \
-    postgresql-client \
-    jq \
-    awscli \
-    curl \
-    libev-dev \
+RUN apt-get update && apt-get install -y wget gnupg
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \
+    apt-key add - \
+    && echo "deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" \
+    >> /etc/apt/sources.list.d/postgresql.list
+
+ADD requirements-apt.txt requirements-test.txt /tmp/
+
+RUN apt-get update \
+    && sed 's/#.*//' /tmp/requirements-apt.txt | xargs apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
-# Upgrade pip first: pip3 install --upgrade pip.
-RUN pip3 install --upgrade pip \
-    && rm -rf $HOME/.cache/pip
+RUN export CPLUS_INCLUDE_PATH=/usr/include/gdal && \
+    export C_INCLUDE_PATH=/usr/include/gdal && \
+    export GDAL_DATA="$(gdal-config --datadir)" && \
+    pip3 install GDAL==$(gdal-config --version)
 
-RUN pip3 install gunicorn flask pyorbital colorama sentry-sdk[flask] raven \
+RUN pip3 install --upgrade pip
+
+RUN pip3 install -r /tmp/requirements-test.txt \
     && rm -rf $HOME/.cache/pip
 
+RUN mkdir -p /code
 WORKDIR /code
-RUN mkdir /code/product-summaries
 
-ADD . .
+ADD setup.py setup.cfg versioneer.py pylintrc Makefile /code/
+ADD cubedash /code/cubedash
+ADD integration_tests /code/integration_tests
+
+RUN pip3 install --upgrade --extra-index-url \
+    https://packages.dea.gadevs.ga/ 'datacube' 'digitalearthau'
+RUN pip3 install -e .[test]
 
-# RUN pip3 install .[deployment]
-RUN pip3 install -U --extra-index-url https://packages.dea.gadevs.ga/ 'datacube>=0.0.dev0' 'digitalearthau>=0.0.dev0';
-RUN pip3 install -e .[test];
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-CMD gunicorn -b '0.0.0.0:8080' -w 1 '--worker-class=egg:meinheld#gunicorn_worker'  --timeout 60 cubedash:app
+CMD flask run --host 0.0.0.0

Makefile

@@ -13,7 +13,7 @@ format:
 
 .PHONY: lint
 lint:
-	python setup.py check -rms
+	python3 setup.py check -rms
 	flake8 cubedash/ integration_tests/
 	black --check cubedash integration_tests
 
@@ -63,10 +63,23 @@ up:
 build:
 	docker-compose build
 
+init-odc:
+	docker-compose exec explorer \
+		datacube system init
+
 schema:
 	docker-compose exec explorer \
 		python3 /code/cubedash/generate.py --init-database
 
 index:
 	docker-compose exec explorer \
-		python3 /code/cubedash/generate.py --all
+		python3 /code/cubedash/generate.py --all
+
+create-test-db-docker:
+	docker-compose exec explorer \
+		bash /code/.docker/create_db.sh
+
+test-docker: build
+	docker-compose --file docker-compose.yml \
+		run explorer \
+			bash -c "make lint; make test"

README.md

@@ -180,5 +180,19 @@ Add a `.datacube_integration.conf` file to your home directory in the same forma
 
 Then run pytest: `pytest integration_tests`
 
-__Warning__ All data in this database will be dropped while running tests. Use a separate one from your normal 
-development db.
+__Warning__ All data in this database will be dropped while running tests. Use a separate one from your normal development db.
+
+## Docker for Development and running tests
+You need to have Docker and Docker Compose installed on your system.
+
+To create your environment, run `make up` or `docker-compose up`.
+
+You need an ODC database, so you'll need to refer to the [ODC docs](https://datacube-core.readthedocs.io/en/latest/) for help on indexing, but you can create the database by running `make initdb` or `docker-compose exec explorer datacube system init`. (This is not enough, you still need to add a product and index datasets.)
+
+When you have some ODC data indexed, you can run `make index` to create the Explorer indexes.
+
+Once Explorer indexes have been created, you can browse the running application at [http://localhost:5000](http://localhost:5000).
+
+You can run tests by first creating a test database `make create-test-db-docker` and then running tests with `make test-docker`.
+
+And you can run a single test in Docker using a command like this: ` docker-compose --file docker-compose.yml run explorer pytest integration_tests/test_dataset_listing.py`

cubedash/_model.py

@@ -20,10 +20,17 @@
 from datacube.index import index_connect
 from datacube.model import DatasetType
 
+
+# Fix up URL Scheme handling using this
+# from https://stackoverflow.com/questions/23347387/x-forwarded-proto-and-flask
+from werkzeug.middleware.proxy_fix import ProxyFix
+
 NAME = "cubedash"
 BASE_DIR = Path(__file__).parent.parent
 
 app = flask.Flask(NAME)
+# Also part of the fix from ^
+app.wsgi_app = ProxyFix(app.wsgi_app)
 
 # Optional environment settings file or variable
 app.config.from_pyfile(BASE_DIR / "settings.env.py", silent=True)

cubedash/_stac.py

@@ -8,13 +8,13 @@
 from typing import Callable, Dict, Iterable, Optional, Sequence, Tuple
 from urllib.parse import urljoin
 
-import flask
 from dateutil.tz import tz
-from flask import abort, request
 
+import flask
 from cubedash.summary._stores import DatasetItem
 from datacube.model import Dataset, Range
 from datacube.utils import DocReader, parse_time
+from flask import abort, request
 
 from . import _model, _utils
 

cubedash/templates/dataset.html

@@ -167,7 +167,7 @@ <h2>Raw Metadata</h2>
             });
             L.control.zoom({position: "bottomright"}).addTo(map);
             map.fitBounds(dataset_data.getBounds(), {animate: false, maxZoom: 5});
-            
+
             window.MAP = map;
         </script>
     {% endif %}

docker-compose.override.yml

@@ -0,0 +1,7 @@
+version: "3.4"
+services:
+  explorer:
+    volumes:
+      # This is in here, because the .override file is default
+      # and I don't want this code dir mounted when I run tests.
+      - ./:/code

docker-compose.yml

@@ -5,32 +5,43 @@ services:
       context: .
       dockerfile: Dockerfile.Dev
     volumes:
-      - ./:/code
-    expose:
-      - 5000
+      - ./.docker/.datacube_integration.conf:/root/.datacube_integration.conf
+    ports:
+      - 5000:5000
     environment:
-      - DB_HOSTNAME=host.docker.internal
+      # - DB_HOSTNAME=host.docker.internal
+      - DB_HOSTNAME=postgres
       - DB_USERNAME=opendatacube
       - DB_PASSWORD=opendatacubepassword
       - DB_DATABASE=opendatacube
       - DB_PORT=5432
       - FLASK_ENV=development
       - FLASK_APP=cubedash
       - FLASK_DEBUG=1
-      - VIRTUAL_HOST=datacube.explorer
-      - VIRTUAL_PORT=5000
-    command: |-
-      bash -c "
-        flask run --host 0.0.0.0
-      "
-  nginx-proxy:
-    image: jwilder/nginx-proxy
-    container_name: nginx-proxy
-    ports:
-      - "5000:80"
-      - "443:443"
-    volumes:
-      - /var/run/docker.sock:/tmp/docker.sock:ro
-      # - ./certs:/etc/nginx/certs
+      # - VIRTUAL_HOST=datacube.explorer
+    command: flask run --host 0.0.0.0
     depends_on:
-      - explorer
+      - postgres
+
+  postgres:
+    image: kartoza/postgis:11.0-2.5
+    environment:
+      - POSTGRES_DB=opendatacube
+      - POSTGRES_PASSWORD=opendatacubepassword
+      - POSTGRES_USER=opendatacube
+    ports:
+      - 5432:5432
+    restart: always
+
+  # Needed for testing HTTPS
+  # nginx-proxy:
+  #   image: jwilder/nginx-proxy
+  #   container_name: nginx-proxy
+  #   ports:
+  #     - "5000:80"
+  #     - "443:443"
+  #   volumes:
+  #     - /var/run/docker.sock:/tmp/docker.sock:ro
+  #     - ./certs:/etc/nginx/certs
+  #   depends_on:
+  #     - explorer

apt-requirements.txt → requirements-apt.txt

@@ -11,7 +11,7 @@ libudunits2-0
 libudunits2-dev
 libxml2-dev
 libxslt-dev
-postgresql-10-postgis-2.4
+postgresql-11-postgis-2.5
 shellcheck
 python3
 python3-pip

requirements-test.txt

@@ -1,14 +1,13 @@
 --extra-index-url https://packages.dea.gadevs.ga/ 
 digitalearthau==20180116+268.g391cbc0
-eodatasets==0.10+40.g759cbb2
 -e git://github.com/maxcountryman/flask-themes@9eac2dcc9b81c3af29c2386ce1afba9b446562bf#egg=Flask-Themes
 affine==2.2.2
 alabaster==0.7.12
 amqp==2.5.1
 appdirs==1.4.3
 aspy.yaml==1.3.0
 atomicwrites==1.3.0
-attrs==19.1.0
+attrs==19.3.0
 Automat==0.7.0
 Babel==2.7.0
 beautifulsoup4==4.8.0
@@ -17,6 +16,7 @@ black==19.3b0
 boltons==19.1.0
 bs4==0.0.1
 cachetools==3.1.1
+cattrs==1.0.0
 celery==4.3.0
 certifi==2019.6.16
 cfgv==2.0.1
@@ -30,14 +30,15 @@ cligj==0.5.0
 cloudpickle==1.2.1
 colorama==0.4.1
 constantly==15.1.0
-coverage==4.5.4
+coverage==5.0.3
 cssselect==1.1.0
-dask
+dask==2.11.0
 dataclasses==0.6
 datacube==1.7
 DAWG==0.7.8
 docopt==0.6.2
 docutils==0.15.2
+eodatasets3==0.6.0
 entrypoints==0.3
 fake-useragent==0.1.11
 Fiona==1.8.6
@@ -65,7 +66,7 @@ mccabe==0.6.1
 mock==3.0.5
 more-itertools==7.2.0
 munch==2.3.2
-netCDF4
+netCDF4==1.5.3
 nodeenv==1.3.3
 nose==1.3.7
 numpy==1.17.0
@@ -93,8 +94,8 @@ pyppeteer==0.0.25
 pyproj==2.2.2
 pyquery==1.4.0
 pyrsistent==0.15.4
-pytest==5.1.0
-pytest-cov==2.7.1
+pytest==5.3.5
+pytest-cov==2.8.1
 python-dateutil==2.8.0
 python-rapidjson==0.8.0
 pytz==2019.2
@@ -103,6 +104,7 @@ rasterio==1.0.25
 redis==3.3.7
 requests==2.22.0
 requests-html==0.10.0
+ruamel.yaml==0.16.10
 scipy==1.3.1
 Shapely==1.6.4.post2
 simplejson==3.16.0