added postgis 10.x, importer, pgdumper for geo/nogeo dbs

GeoNode · Jun 13, 2019 · 1be8c64 · 1be8c64
1 parent d35e097
commit 1be8c64
Show file tree

Hide file tree

Showing 7 changed files with 119 additions and 12 deletions.
diff --git a/scripts/spcgeonode/.env b/scripts/spcgeonode/.env
@@ -18,16 +18,30 @@ HTTPS_HOST=
 HTTPS_PORT=443
 
 # IP or domain name and port where the server can be reached on HTTP (leave HOST empty if you want to use HTTPS only)
-HTTP_HOST=127.0.0.1
+HTTP_HOST=172.30.218.218
+#HTTP_HOST=127.0.0.1
 HTTP_PORT=80
 
 # Email where alters should be sent. This will be used by let's encrypt and as the django admin email.
 ADMIN_USERNAME=super
 ADMIN_PASSWORD=duper
 ADMIN_EMAIL=admin@example.com
 
-# PostgreSQL superuser password (to be set if PostgreSQL is exposed)
-POSTGRES_PASSWORD=
+# Database environment
+POSTGRES_USER=postgres
+## PostgreSQL superuser password (to be set if PostgreSQL is exposed)
+POSTGRES_PASSWORD=postgres
+## geonode specific variables
+## the standart geonode django database with non-spatial data
+## the db name and username are identical by design
+GEONODE_DATABASE=geonode
+GEONODE_DATABASE_USER=geonode
+GEONODE_DATABASE_PASSWORD=geonode
+## the spatial geonode django postigs database for data ingestion
+## the db name and username are identical by design
+GEONODE_GEODATABASE=geonode_data
+GEONODE_GEODATABASE_USER=geonode_data
+GEONODE_GEODATABASE_PASSWORD=geonode
 
 # Django secret key (replace this by any complex and random string)
 SECRET_KEY=1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ

diff --git a/scripts/spcgeonode/docker-compose.yml b/scripts/spcgeonode/docker-compose.yml
@@ -22,10 +22,23 @@ x-common-django:
     - TIME_ZONE=${TIME_ZONE}
     - ALLOWED_HOSTS=['nginx','127.0.0.1','localhost','$HTTPS_HOST','$HTTP_HOST']
     - SECRET_KEY=${SECRET_KEY}
+    # data
+    ## -- comment to disable postgis ingestion of uploaded shapefiles
+    - DEFAULT_BACKEND_UPLOADER=geonode.importer
+    - DEFAULT_BACKEND_DATASTORE=datastore
+    # database info, ${variables} are read from .env
+    - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    - GEONODE_DATABASE=${GEONODE_DATABASE}
+    - GEONODE_DATABASE_PASSWORD=${GEONODE_DATABASE_PASSWORD}
+    - GEONODE_GEODATABASE=${GEONODE_GEODATABASE}
+    - GEONODE_GEODATABASE_PASSWORD=${GEONODE_GEODATABASE_PASSWORD}
+    - DATABASE_URL=postgres://${GEONODE_DATABASE}:${GEONODE_DATABASE_PASSWORD}@postgres:5432/${GEONODE_DATABASE}
+    - GEODATABASE_URL=postgis://${GEONODE_GEODATABASE}:${GEONODE_GEODATABASE_PASSWORD}@postgres:5432/${GEONODE_GEODATABASE}
+
+
     # hardcoded
     - DEBUG=False
     - DJANGO_SETTINGS_MODULE=geonode.settings
-    - DATABASE_URL=postgres://postgres:${POSTGRES_PASSWORD}@postgres:5432/postgres
     - BROKER_URL=amqp://rabbitmq:5672
     - STATIC_ROOT=/spcgeonode-static/
     - MEDIA_ROOT=/spcgeonode-media/
@@ -110,7 +123,8 @@ services:
       - HTTP_PORT=${HTTP_PORT}
       - ADMIN_USERNAME=${ADMIN_USERNAME}
       - ADMIN_PASSWORD=${ADMIN_PASSWORD}
-      - DATABASE_URL=postgres://postgres:${POSTGRES_PASSWORD}@postgres:5432/postgres
+      - DATABASE_URL=postgres://${GEONODE_DATABASE}:${GEONODE_DATABASE_PASSWORD}@postgres:5432/${GEONODE_DATABASE}
+      - GEODATABASE_URL=postgis://${GEONODE_GEODATABASE}:${GEONODE_GEODATABASE_PASSWORD}@postgres:5432/${GEONODE_GEODATABASE}      
     volumes:
       - geodatadir:/spcgeonode-geodatadir/
     restart: on-failure
@@ -128,14 +142,30 @@ services:
       - certificates:/spcgeonode-certificates/
     restart: on-failure
 
-  pgdumper:
-    image: olivierdalang/spcgeonode:pgdumper-latest
+  # dumps the non geodata db
+  pgdumper_nogeo:
+    container_name: dumper_${GEONODE_DATABASE}_${COMPOSE_PROJECT_NAME}
+    image: tisdar/pgdumper:latest
     build: ./pgdumper/
+    volumes:
+      - pgdumps:/spcgeonode-pgdumps/
+    restart: on-failure
     environment:
-      - DATABASE_URL=postgres://postgres:${POSTGRES_PASSWORD}@postgres:5432/postgres
+      - DATABASE=${GEONODE_DATABASE}
+      - DATABASE_URL=postgres://${GEONODE_DATABASE}:${GEONODE_DATABASE_PASSWORD}@postgres:5432/${GEONODE_DATABASE}
+
+  # dumps the geodata db
+  pgdumper_geo:
+    container_name: dumper_${GEONODE_GEODATABASE}_${COMPOSE_PROJECT_NAME}
+    image: tisdar/pgdumper:latest
+    build: ./pgdumper/
     volumes:
       - pgdumps:/spcgeonode-pgdumps/
     restart: on-failure
+    environment:
+      - DATABASE=${GEONODE_GEODATABASE}
+      # pgdumper needs postgres:// as connection string for the geodatabase
+      - DATABASE_URL=postgres://${GEONODE_GEODATABASE}:${GEONODE_GEODATABASE_PASSWORD}@postgres:5432/${GEONODE_GEODATABASE} 
 
   rclone:
     image: olivierdalang/spcgeonode:rclone-latest
@@ -153,12 +183,24 @@ services:
 
   # PostGIS database.
   postgres:
-    image: mdillon/postgis:9.6-alpine
+    # use geonode official postgis 10 image
+    image: geonode/postgis:10
+    build: ./postgis/
     environment:
+      # database info, ${variables} are read from .env
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - GEONODE_DATABASE=${GEONODE_DATABASE}
+      - GEONODE_DATABASE_PASSWORD=${GEONODE_DATABASE_PASSWORD}
+      - GEONODE_GEODATABASE=${GEONODE_GEODATABASE}
+      - GEONODE_GEODATABASE_PASSWORD=${GEONODE_GEODATABASE_PASSWORD}
+      - DATABASE_URL=postgres://${GEONODE_DATABASE}:${GEONODE_DATABASE_PASSWORD}@postgres:5432/${GEONODE_DATABASE}
+      - GEODATABASE_URL=postgis://${GEONODE_GEODATABASE}:${GEONODE_GEODATABASE_PASSWORD}@postgres:5432/${GEONODE_GEODATABASE}
     volumes:
       - database:/var/lib/postgresql/data/
     restart: on-failure
+    # uncomment to enable remote connections to postgres
+    #ports:
+    #  - "5432:5432"
 
   # Vanilla RabbitMQ service. This is needed by celery
   rabbitmq:

diff --git a/scripts/spcgeonode/pgdumper/Dockerfile b/scripts/spcgeonode/pgdumper/Dockerfile
@@ -1,5 +1,6 @@
-FROM python:2.7.13-alpine3.6
-
+FROM python:3.6.8-alpine3.8
+# alpine 3.8 uses postgres 10.x
+
 # 1-2. Install system dependencies (we only need the pg_dump binary from postgresql, other dependencies are in postgresql-client)
 RUN apk add --no-cache postgresql-client && \
     apk add --no-cache --virtual BUIID_DEPS postgresql && \

diff --git a/scripts/spcgeonode/pgdumper/crontab b/scripts/spcgeonode/pgdumper/crontab
@@ -5,4 +5,4 @@
 # │     │     │     │     ┌───────────── day of week (0 - 6) (Sunday to Saturday; 7 is also Sunday on some systems)
 # │     │     │     │     │
 
-  0     0     *     *     *     date && pg_dump -C "${DATABASE_URL}" > /spcgeonode-pgdumps/latest.pgdump && echo "Dump successful"
+  0     0     *     *     *     date && pg_dump -C "${DATABASE_URL}" > /spcgeonode-pgdumps/${DATABASE}_daily.pgdump && echo "${DATABASE} dump successful"
diff --git a/scripts/spcgeonode/pgdumper/docker-entrypoint.sh b/scripts/spcgeonode/pgdumper/docker-entrypoint.sh
@@ -1,6 +1,7 @@
 #!/bin/sh
 
 envsubst '\$DATABASE_URL' < crontab.envsubst > crontab
+envsubst '\$DATABASE' < crontab.envsubst > crontab
 /usr/bin/crontab crontab
 rm crontab crontab.envsubst
 

diff --git a/scripts/spcgeonode/postgis/Dockerfile b/scripts/spcgeonode/postgis/Dockerfile
@@ -0,0 +1,4 @@
+FROM mdillon/postgis:10
+MAINTAINER GeoNode development team
+
+COPY ./initdb-geonode.sh /docker-entrypoint-initdb.d/geonode.sh
diff --git a/scripts/spcgeonode/postgis/initdb-geonode.sh b/scripts/spcgeonode/postgis/initdb-geonode.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+
+function create_geonode_user_and_database() {
+	local db=$1
+	echo "  Creating user and database '$db'"
+	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+	    CREATE USER $db;
+		ALTER USER $db with encrypted password '$GEONODE_DATABASE_PASSWORD';
+	    CREATE DATABASE $db;
+	    GRANT ALL PRIVILEGES ON DATABASE $db TO $db;
+EOSQL
+}
+
+function create_geonode_user_and_geodatabase() {
+	local geodb=$1
+	echo "  Creating user and database '$geodb'"
+	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+	    CREATE USER $geodb;
+		ALTER USER $geodb with encrypted password '$GEONODE_GEODATABASE_PASSWORD';
+	    CREATE DATABASE $geodb;
+	    GRANT ALL PRIVILEGES ON DATABASE $geodb TO $geodb;
+EOSQL
+}
+
+function update_geodatabase_with_postgis() {
+	local geonode_data=$1
+	echo "  Updating geodatabase '$geonode_data' with extension"
+	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$geonode_data" <<-EOSQL
+		CREATE EXTENSION postgis;
+EOSQL
+}
+
+if [ -n "$GEONODE_DATABASE" ]; then
+	echo "Geonode database creation requested: $GEONODE_DATABASE"
+	create_geonode_user_and_database $GEONODE_DATABASE
+	echo "Geonode database created"
+fi
+
+if [ -n "$GEONODE_GEODATABASE" ]; then
+	echo "Geonode geodatabase creation requested: $GEONODE_GEODATABASE"
+	create_geonode_user_and_geodatabase $GEONODE_GEODATABASE
+	update_geodatabase_with_postgis $GEONODE_GEODATABASE
+	echo "Geonode geodatabase created"
+fi