Updated docker script for adaguc-services

KNMI · Mar 16, 2018 · 90cd1c2 · 90cd1c2
1 parent 23dc860
commit 90cd1c2
Show file tree

Hide file tree

Showing 14 changed files with 599 additions and 125 deletions.
diff --git a/docker/README.md → Docker/README.md b/docker/README.md → Docker/README.md
diff --git a/Docker/adaguc-server-config.xml b/Docker/adaguc-server-config.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Configuration>
+  <CacheDocs enabled="false"/>
+  <TempDir value="{ADAGUC_TMP}"/>
+  <Path value="{ADAGUC_PATH}/data"/>
+  <DataBase parameters="host=localhost port=5432 user=adaguc password=adaguc dbname=adaguc"/>
+
+
+  <!-- WMS settings -->
+  <WMS>
+    <Title>ADAGUC WMS Service</Title>
+    <Abstract>This service demonstrates how the ADAGUC server can be used to create OGC services.</Abstract>
+    <RootLayer>
+      <Title>Root Layer Title</Title>
+      <Abstract>Root Layer Abstract</Abstract>
+    </RootLayer>
+
+  </WMS>
+
+
+  <!-- WMS -->
+  <Include location="{ADAGUC_PATH}/data/config/includes/WMS.include.xml"/>
+
+  <!-- WCS settings -->
+  <WCS>
+    <Title>ADAGUC WCS Service</Title>
+    <Label>wcsLabel</Label>
+  </WCS>
+
+  <!-- WCSFormat -->
+  <Include location="{ADAGUC_PATH}/data/config/includes/WCSFormat.include.xml"/>
+
+  <!-- Projections -->
+  <Include location="{ADAGUC_PATH}/data/config/includes/Projection.include.xml"/>
+
+  <!--Legends -->
+  <Include location="{ADAGUC_PATH}/data/config/includes/DefaultStyles.include.xml"/>
+
+  <AutoResource enableautoopendap="true" enablelocalfile="true" enablecache="false">
+    <Dir basedir="/data/adaguc-autowms" prefix="/data/adaguc-autowms/"/>
+    <ImageText attribute="source"></ImageText>
+  </AutoResource>
+
+  <Dataset enabled="true" location="/data/adaguc-datasets/"/>
+  <Dataset enabled="true" location="/data/adaguc-datasets-internal/"/>
+
+  <OpenDAP enabled="true" path="adaguc-services/adagucopendap"/>
+  <!-- End of configuration /-->
+</Configuration>
diff --git a/Docker/adaguc-server-createtiles.sh b/Docker/adaguc-server-createtiles.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+export ADAGUC_PATH=/adaguc/adaguc-server-master/
+export ADAGUC_TMP=/tmp
+
+if [[ $1 ]]; then
+
+  # Update a specific dataset
+  for configfile in /data/adaguc-datasets/$1.xml ;do
+    filename=/data/adaguc-datasets/"${configfile##*/}" 
+    echo "Starting update for ${filename}" 
+    /adaguc/adaguc-server-master/bin/adagucserver --updatedb --config /adaguc/adaguc-server-config.xml,${filename}
+    /adaguc/adaguc-server-master/bin/adagucserver --createtiles --config /adaguc/adaguc-server-config.xml,${filename}
+  done
+
+else
+  echo "Please specify a dataset"
+fi
diff --git a/Docker/adaguc-server-logrotate b/Docker/adaguc-server-logrotate
@@ -0,0 +1,8 @@
+/var/log/adaguc/*log {
+    su root root
+    daily
+    rotate 10
+    size 100k
+    missingok
+    notifempty
+}
diff --git a/Docker/adaguc-server-updatedatasets.sh b/Docker/adaguc-server-updatedatasets.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+export ADAGUC_PATH=/adaguc/adaguc-server-master/
+export ADAGUC_TMP=/tmp
+
+if [[ $1 ]]; then
+
+  # Update a specific dataset
+  for configfile in /data/adaguc-datasets/$1.xml ;do
+    filename=/data/adaguc-datasets/"${configfile##*/}" 
+    echo "Starting update for ${filename}" 
+    /adaguc/adaguc-server-master/bin/adagucserver --updatedb --config /adaguc/adaguc-server-config.xml,${filename}
+  done
+
+else
+
+  # Update all datasets
+  for configfile in /data/adaguc-datasets/*xml ;do
+    filename=/data/adaguc-datasets/"${configfile##*/}" 
+    echo "Starting update for ${filename}" 
+    /adaguc/adaguc-server-master/bin/adagucserver --updatedb --config /adaguc/adaguc-server-config.xml,${filename}
+  done
+
+fi
diff --git a/Docker/adaguc-services-config.xml b/Docker/adaguc-services-config.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<adaguc-services>
+  <external-home-url>{ENV.EXTERNALADDRESS}/adaguc-services/</external-home-url>
+  <userworkspace>/adaguc/userworkspace</userworkspace>
+  <basedir>/adaguc/basedir</basedir>
+  <server>
+    <port>8080</port>
+  </server>
+  <adaguc-server>
+    <adagucexecutable>/adaguc/adaguc-server-master/bin/adagucserver</adagucexecutable>
+    <export>ADAGUC_PATH=/adaguc/adaguc-server-master/</export>
+    <export>ADAGUC_TMP=/tmp</export>
+    <export>ADAGUC_CONFIG=/adaguc/adaguc-server-config.xml</export>
+    <export>ADAGUC_DATARESTRICTION=FALSE</export>
+    <export>ADAGUC_ENABLELOGBUFFER=TRUE</export>
+    <export>ADAGUC_LOGFILE=/var/log/adaguc/adaguc-server.log</export>
+    <export>ADAGUC_FONT=/adaguc/adaguc-server-master/data/fonts/FreeSans.ttf</export>
+    <export>ADAGUC_ONLINERESOURCE={ENV.EXTERNALADDRESS}/adaguc-services/adagucserver?</export>
+  </adaguc-server>
+  <autowms>
+    <enabled>true</enabled>
+    <autowmspath>/data/adaguc-autowms/</autowmspath>
+    <datasetpath>/data/adaguc-datasets/</datasetpath>
+  </autowms>
+</adaguc-services>
diff --git a/Docker/baselayers.xml b/Docker/baselayers.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Configuration>
+  <Legend name="baselayer" type="interval">
+    <palette min="0" max="240" red="0"  green="0" blue="0"/>
+  </Legend>
+
+  <Style name="baselayer">
+    <Legend fixedclasses="true" tickinterval="1000" tickround="1">baselayer</Legend>
+    <Min>0.0</Min>
+    <Max>1000</Max>
+    <RenderMethod>nearest</RenderMethod>
+    <NameMapping name="nearest"   title="Land sea mask" abstract="Land sea mask"/>
+    <FeatureInterval match=".*" matchid="abbrev" bgcolor="#CCCCFF" fillcolor="#CCFFCC"/>
+  </Style>
+
+  <Style name="overlay">
+    <Legend fixedclasses="true" tickinterval="1000" tickround="1">baselayer</Legend>
+    <Min>0.0</Min>
+    <Max>1000</Max>
+    <RenderMethod>polyline</RenderMethod>
+    <NameMapping name="polyline"   title="Overlay countries" abstract="Overlay countries"/>
+    <FeatureInterval match=".*" matchid="abbrev" bordercolor="#000000" borderwidth="1" linecolor="#000000" linewidth="1"/>
+  </Style>
+
+  <Layer>
+      <Name>baselayer</Name>
+      <Title>baselayer</Title>
+      <FilePath>{ADAGUC_PATH}/data/datasets/countries.geojson</FilePath>
+      <Variable>features</Variable>
+      <Styles>baselayer</Styles>
+  </Layer>
+
+  <Layer>
+      <Name>overlay</Name>
+      <Title>overlay</Title>
+      <FilePath>{ADAGUC_PATH}/data/datasets/countries.geojson</FilePath>
+      <Variable>features</Variable>
+      <Styles>overlay</Styles>
+  </Layer>
+</Configuration>  
diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml
@@ -0,0 +1,42 @@
+version: '2'
+
+services:
+    adaguc-viewer:
+        image: openearth/adaguc-viewer
+        container_name: adaguc-viewer
+        ports:
+            - "8091:80"
+        environment:
+            - "LOCAL_ADAGUCSERVER_ADDR=http://localhost:8090/" # Should be same as adaguc-server's EXTERNALADDRESS
+            - "REMOTE_ADAGUCSERVER_ADDR=http://adaguc-server:8080/" # Do not change this within the docker-compose environment
+            - "ADAGUCSERVICES_AUTOWMS=http://localhost:8090/adaguc-services/autowms?" 
+    adaguc-server:
+        image: openearth/adaguc-server
+        container_name: adaguc-server
+        ports:
+            - "8090:8080"
+        volumes:
+            - $HOME/adaguc-server-docker/adaguc-datasets:/data/adaguc-datasets
+            - $HOME/adaguc-server-docker/adaguc-autowms:/data/adaguc-autowms
+            - $HOME/adaguc-server-docker/adaguc-data:/data/adaguc-data
+            - $HOME/adaguc-server-docker/adagucdb:/adaguc/adagucdb
+            - $HOME/adaguc-server-docker/adaguc-logs:/var/log/adaguc
+        environment:
+            - "EXTERNALADDRESS=http://localhost:8090/"
+
+#cd ./adaguc-server/
+#docker pull openearth/adaguc-viewer
+#Either do:
+#docker pull openearth/adaguc-server #<-- Pull docker image from dockerhub, or do docker build -t openearth/adaguc-server . 
+
+#mkdir -p $HOME/adaguc-server-docker/adaguc-data
+#mkdir -p $HOME/adaguc-server-docker/adaguc-datasets
+#mkdir -p $HOME/adaguc-server-docker/adaguc-autowms
+#mkdir -p $HOME/adaguc-server-docker/adagucdb && chmod 777 $HOME/adaguc-server-docker/adagucdb
+#mkdir -p $HOME/adaguc-server-docker/adaguc-logs && chmod 777 $HOME/adaguc-server-docker/adaguc-logs
+
+# docker-compose -f ./Docker/docker-compose.yml up 
+# Go to http://localhost:8091/adaguc-viewer/ or http://localhost:8090/adaguc-services/wms.cgi? 
+
+# CTRL+C and docker-compose down # To stop 
+
diff --git a/Docker/start.sh b/Docker/start.sh
@@ -0,0 +1,175 @@
+#!/bin/bash
+echo ADAGUC_SERVICES_HOME=${ADAGUC_SERVICES_HOME}
+
+ADAGUC_SERVICES_SECURITY=${ADAGUC_SERVICES_HOME}/security
+
+# Setup dirs
+mkdir -p ${ADAGUC_SERVICES_HOME}/data/adaguc-services-base
+mkdir -p ${ADAGUC_SERVICES_HOME}/data/adaguc-services-space
+mkdir -p ${ADAGUC_SERVICES_SECURITY}
+mkdir -p ${ADAGUC_SERVICES_HOME}/.globus/certificates
+mkdir -p ${ADAGUC_SERVICES_HOME}/adaguc-services-tmp
+mkdir -p ${ADAGUC_SERVICES_HOME}/wpsoutputs
+
+### Setup truststore and keystore ###
+
+# Get configured hostname
+HOSTNAME=${EXTERNALADDRESS}
+
+# Remove protocol part of url  #
+HOSTNAME="${HOSTNAME#http://}"
+HOSTNAME="${HOSTNAME#https://}"
+HOSTNAME="${HOSTNAME#ftp://}"
+HOSTNAME="${HOSTNAME#scp://}"
+HOSTNAME="${HOSTNAME#scp://}"
+HOSTNAME="${HOSTNAME#sftp://}"
+
+# Remove username and/or username:password part of URL  #
+HOSTNAME="${HOSTNAME#*:*@}"
+HOSTNAME="${HOSTNAME#*@}"
+
+# Remove rest of urls #
+HOSTNAME=${HOSTNAME%%/*}
+HOSTNAME=${HOSTNAME%%:*}
+
+# Show domain name only #
+echo "Derived hostname is $HOSTNAME"
+
+export KEYSTOREUSERID=$(stat -c "%u" ${ADAGUC_SERVICES_SECURITY})
+echo "Got ${KEYSTOREUSERID} from owner of dir ${ADAGUC_SERVICES_SECURITY}"
+# Create security user
+echo "Using KEYSTOREUSERID : ${KEYSTOREUSERID}"
+useradd --shell /bin/bash -u ${KEYSTOREUSERID} -o -c "" -m useradaguc
+
+# If needed create a self signed certificate in a keystore for serving over HTTPS
+if [ ! -f ${ADAGUC_SERVICES_SECURITY}/keystore.jks ]; then
+  echo "Generating self signed certificate for HTTPS with hostname ${HOSTNAME}"
+  runuser -l useradaguc -c "keytool -genkey -noprompt -keypass password -alias tomcat \
+  -keyalg RSA -storepass password -keystore ${ADAGUC_SERVICES_SECURITY}/keystore.jks -deststoretype pkcs12 \
+  -dname CN=${HOSTNAME}"
+else
+  echo "Using provided certificate for HTTPS"
+fi
+
+# If needed create a truststore based on java truststore
+if [ ! -f ${ADAGUC_SERVICES_SECURITY}/truststore.ts ]; then
+  echo "Using default truststore from /etc/pki/java/cacerts"
+  runuser -l useradaguc -c "cp /etc/pki/java/cacerts ${ADAGUC_SERVICES_SECURITY}/truststore.ts"
+fi
+
+### Make sure that this service trusts itself by adding its certificate to the trust store ###
+
+# 1) Export certificate from a keystore to a file called adaguc-services-cert.pem
+keytool -export -alias tomcat -rfc -file adaguc-services-cert.pem -keystore ${ADAGUC_SERVICES_SECURITY}/keystore.jks -storepass password
+
+# 2) Put this certificate from adaguc-services-cert.pem into the truststore
+keytool -delete -alias adagucservicescert -keystore ${ADAGUC_SERVICES_SECURITY}/truststore.ts -storepass changeit -noprompt
+keytool -import -v -trustcacerts -alias adagucservicescert -file adaguc-services-cert.pem -keystore ${ADAGUC_SERVICES_SECURITY}/truststore.ts -storepass changeit -noprompt
+
+
+# Create CA for tokenapi: file and key for  authority /O=KNMI/OU=RDWDT/CN=adaguc-services_ca_tokenapi"
+
+if [ ! -f ${ADAGUC_SERVICES_SECURITY}/adaguc-services-ca.cert ]; then
+
+openssl req \
+    -new \
+    -newkey rsa:4096 \
+    -days 365 \
+    -nodes \
+    -x509 \
+    -subj "/O=KNMI/OU=RDWDT/CN=adaguc-services_ca_tokenapi" \
+    -keyout ${ADAGUC_SERVICES_SECURITY}/adaguc-services-ca.key \
+    -out ${ADAGUC_SERVICES_SECURITY}/adaguc-services-ca.cert
+
+# Put this CA in the truststore
+
+keytool -delete -alias adaguc-services-ca -keystore ${ADAGUC_SERVICES_SECURITY}/truststore.ts -storepass changeit -noprompt
+keytool -import -v -trustcacerts -alias adaguc-services-ca -file ${ADAGUC_SERVICES_SECURITY}/adaguc-services-ca.cert -keystore ${ADAGUC_SERVICES_SECURITY}/truststore.ts -storepass changeit -noprompt
+else
+  echo "Using CA file ${ADAGUC_SERVICES_SECURITY}/adaguc-services-ca.cert"
+fi
+
+HOME=$ADAGUC_SERVICES_HOME
+
+
+### Configure postgres ###
+
+# Detect postgres user id
+PGUSERNAME=userpostgres
+if [ -z ${PGUSERID+x} ] || [ -z ${PGUSERID} ]; then 
+  echo "PGUSERID is unset, trying to get id from directory"; 
+  export PGUSERID=$(stat -c "%u" ${ADAGUCDB})
+  echo "Got ${PGUSERID} from owner of dir ${ADAGUCDB}"
+  if [ ${PGUSERID} == 0 ]; then 
+    echo "PGUSERID has root id, setting to postgres"; 
+    PGUSERNAME=postgres
+    export PGUSERID=`id -u postgres`
+  fi
+else 
+  echo "PGUSERID is set to '$PGUSERID'"; 
+fi
+
+if [ ${KEYSTOREUSERID} -ne ${PGUSERID} ]; then
+  # Create postgres user
+  echo "Using PGUSERID : ${PGUSERID}"
+  useradd --shell /bin/bash -u ${PGUSERID} -o -c "" -m $PGUSERNAME
+  export HOME=/home/$PGUSERNAME
+else 
+  # Same user as security folder
+  PGUSERNAME=useradaguc
+fi
+
+# Set postgres permissions
+chmod 777 /var/run/postgresql/
+runuser -l $PGUSERNAME -c "touch /var/log/adaguc/postgresql.log"
+runuser -l $PGUSERNAME -c "chmod 777 /var/log/adaguc/postgresql.log"
+chown $PGUSERNAME ${ADAGUCDB}
+runuser -l $PGUSERNAME -c "chmod 700 ${ADAGUCDB}"
+
+# Check if a db already exists for given path
+dbexists=`runuser -l $PGUSERNAME -c "(ls ${ADAGUCDB}/postgresql.conf >> /dev/null 2>&1 && echo yes) || echo no"`
+if [ ${dbexists} == "no" ]
+then
+  echo "Initializing new postgresql database"
+  #mkdir -p ${ADAGUCDB} && chmod 777 ${ADAGUCDB} && chown postgres: ${ADAGUCDB} && #TODO NOT NEEDED ANYMORE?
+  runuser -l $PGUSERNAME -c "pg_ctl initdb -U adaguc -w -D ${ADAGUCDB}" && \
+  runuser -l $PGUSERNAME -c "pg_ctl -w -U adaguc -D ${ADAGUCDB} -l /var/log/adaguc/postgresql.log start" && \
+  echo "Configuring new postgresql database" && \
+  runuser -l $PGUSERNAME -c "createuser --superuser adaguc" && \
+  runuser -l $PGUSERNAME -c "psql -U adaguc postgres -c \"ALTER USER adaguc PASSWORD 'adaguc';\"" && \
+  runuser -l $PGUSERNAME -c "psql -U adaguc postgres -c \"CREATE DATABASE adaguc;\""
+
+  if [ $? -ne 0 ]
+  then
+  exit 1
+  fi
+else 
+  echo "Re-using persistent postgresql database from ${ADAGUCDB}" && \
+  runuser -l $PGUSERNAME -c "pg_ctl -w -U adaguc -D ${ADAGUCDB} -l /var/log/adaguc/postgresql.log start"
+  if [ $? -ne 0 ]
+  then
+  exit 1
+  fi
+fi
+
+echo "Checking POSTGRESQL DB" &&  runuser -l $PGUSERNAME -c "psql -U adaguc postgres -c \"show data_directory;\""
+if [ $? -ne 0 ]
+then
+  echo "Unable to connect to postgres database"
+  exit 1
+fi  
+
+### Update baselayers and check if this succeeds ###
+export ADAGUC_PATH=/adaguc/adaguc-server-master/ && \
+export ADAGUC_TMP=/tmp && \
+/adaguc/adaguc-server-master/bin/adagucserver --updatedb \
+  --config /adaguc/adaguc-server-config.xml,baselayers.xml
+
+if [ $? -ne 0 ]
+then
+  echo "Unable to update baselayers with adaguc-server --updatedb"
+  exit 1
+fi  
+
+echo "Starting TOMCAT Server" && \
+java -jar /adaguc/adaguc-services.war