working Docker for ETL with PostGIS!!

docker/README.md

@@ -0,0 +1,13 @@
+# Docker
+
+Below the Docker image recipes.
+
+## Shorthands
+
+Run postgis
+
+sudo docker run --name "postgis" -p 5432:5432 -d -t kartoza/postgis:9.4-2.1
+psql -h localhost -U docker  -l
+
+Remove images
+sudo docker rm -v $(sudo docker ps -a -q -f status=exited)

docker/postgis/Dockerfile

@@ -1,44 +1,3 @@
 FROM ubuntu:14.04
-# FROM python:2.7
 
-MAINTAINER just@justobjects.nl
-
-# Use local cached debs from host (saves your bandwidth!)
-# Change ip below to that of your apt-cacher-ng host
-# Or comment this line out if you do not with to use caching
-# ADD 71-apt-cacher-ng /etc/apt/apt.conf.d/71-apt-cacher-ng
-RUN export DEBIAN_FRONTEND=noninteractive TERM=linux && \
-  echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-
-RUN apt-get -y update
-RUN apt-get install -y software-properties-common
-RUN apt-add-repository ppa:ubuntugis/ubuntugis-unstable
-RUN apt-get -y update
-
-#-------------Application Specific Stuff ----------------------------------------------------
-
-#     python-imaging \
-#      python-yaml \
-#    libjpeg-dev \
-#    zlib1g-dev \
-#    libfreetype6-dev \
-#    python-virtualenv
-
-RUN apt-get install -y \
-	git \
-    libproj0 \
-    libgeos-dev \
-    python-lxml \
-    python-gdal \
-    libgdal-dev \
-    build-essential \
-    python-dev \
-    python-setuptools
-
-# ADD start.sh /start.sh
-# RUN chmod 0755 /start.sh
-RUN git clone https://github.com/justb4/stetl /opt/stetl
-
-RUN cd /opt/stetl; python setup.py install
-
-ENTRYPOINT ["/usr/local/bin/stetl"]
+# TODO for now kartoza/postgis:9.4-2.1 fulfills all needs

docs/platform/installation.rst

@@ -333,3 +333,19 @@ PostGIS from Kartoza, see https://hub.docker.com/r/kartoza/postgis/ and https://
     template_postgis | postgres | UTF8      | C       | C     |
    (5 rows)
 
+ETL - Last Measurements
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Uses the ``geonovum/stetl`` image with Stetl config from GitHub.  ::
+
+   # build stetl image
+   cd ~/git/docker/stetl
+   sudo docker build -t geonovum/stetl .
+
+   # run last measurements ETL, linking to postgis image
+   cd ~/git/etl
+   ./last.sh
+
+   # may first do ./db-init.sh to create DB schema and tables
+
+

etl/README.md

@@ -1,11 +1,9 @@
+# ETL - Extract, Transform, Load for sensor data
+
 Sources for ETL of Smart Emission project Nijmegen.
 
 Uses host-specific variables for databases, passwords etc.
-To use define a file on your host called options-<your host>.sh
-Take the example file options-sunda.sh for initial values and adapt to your host.
 
 Most ETL is developed using Stetl: http://stetl.org
 
 last.sh: last values
-
-

etl/db-init.sh

@@ -0,0 +1,5 @@
+# passwd docker
+
+export PGPASSWORD=docker
+
+psql -h localhost -U docker gis -f db/db-schema-last.sql

etl/last.cfg

@@ -11,7 +11,7 @@ chains = input_raw_sensor_last_api|output_postgres_insert
 
 # for reading files from CityGIS Sensor REST API
 [input_raw_sensor_last_api]
-class = py.rawsensorlastinput.RawSensorLastInput
+class = rawsensorlastinput.RawSensorLastInput
 url = {raw_device_url}
 
 # for testing/debugging

etl/last.sh

@@ -3,4 +3,8 @@
 # ETL for reading last values from Smart Emission CityGIS Raw Sensor API
 #
 
-sudo docker run --link postgis2:postgis2 -v `pwd`:`pwd` -w `pwd`  -t -i geonovum/stetl -c last.cfg -a options/docker.args
+STETL_ARGS="-c last.cfg -a options/docker.args"
+WORK_DIR="`pwd`"
+PG_HOST=postgis
+
+sudo docker run --link ${PG_HOST}:${PG_HOST} -v ${WORK_DIR}:${WORK_DIR} -w ${WORK_DIR} -t -i geonovum/stetl ${STETL_ARGS}

etl/options/docker.args

@@ -2,8 +2,8 @@
 
 raw_device_url=http://whale.citygis.nl/sensors/v1
 
-# pg_host=172.17.0.1
-pg_host=postgis2
+# postgis docker container IP is mapped by docker via --link option
+pg_host=postgis
 pg_user=docker
 pg_password=docker
 pg_port=5432