more fixes on the docker

linkedgeodata-cli/bin/lgd-run-sync.sh

@@ -4,8 +4,8 @@ configFileName="$1"
 
 #osmosis={osmosis:-osmosis}
 osmosis="osmosis"
-targetPath="sync"
-sleepInterval="60"
+targetPath=${targetPath:-"sync"}
+OSM_DATA_SYNC_RECHECK_INTERVAL=${OSM_DATA_SYNC_RECHECK_INTERVAL:-60}
 
 log() { echo "[`date +'%Y-%m-%d %H:%M:%S'`] $$ [$1] $2" ; }
 
@@ -62,8 +62,9 @@ applyChanges
 while [ 1 ]; do
     "$osmosis" --read-replication-interval workingDirectory=$targetPath --simplify-change --write-xml-change "$targetPath/diff.osc"
     applyChanges
-    echo "Going to sleep for $sleepInterval seconds..."
-    sleep "$sleepInterval"
+    # TODO Enhance script to compute time duration to next recheck interval based on timestamp in latest state.txt file
+    echo "Going to sleep for $OSM_DATA_SYNC_RECHECK_INTERVAL seconds..."
+    sleep "$OSM_DATA_SYNC_RECHECK_INTERVAL"
 done
 
 

linkedgeodata-docker/.env

@@ -22,7 +22,10 @@ DB_MAINTENANCE_WORK_MEM=256MB
 OSM_DATA_BASE_URL=http://downloads.linkedgeodata.org/debugging/monaco-170618.osm.pbf
 OSM_DATA_SYNC_URL=http://download.geofabrik.de/europe/monaco-updates/
 
+# Crunch interval is the interval for which to merge individual updates into a single larger update
+OSM_DATA_SYNC_CRUNCH_INTERVAL=86400
 
+# These constants should not be needed as they can be obtained automatically by analysing the osm data base url
 #OSM_DATA_SYNC_RECHECK_INTERVAL=900
 #OSM_DATA_SYNC_UPDATE_INTERVAL=3600
 

linkedgeodata-docker/docker-compose.yml

@@ -36,6 +36,7 @@ services:
     environment:
       - OSM_DATA_BASE_URL=${OSM_DATA_BASE_URL}
       - OSM_DATA_SYNC_URL=${OSM_DATA_SYNC_URL}
+      - OSM_DATA_SYNC_CRUNCH_INTERVAL=${OSM_DATA_SYNC_CRUNCH_INTERVAL}
     depends_on:
       lgd-db:
         condition: service_healthy
@@ -68,6 +69,7 @@ services:
     environment:
       - OSM_DATA_BASE_URL=${OSM_DATA_BASE_URL}
       - OSM_DATA_SYNC_URL=${OSM_DATA_SYNC_URL}
+      - OSM_DATA_SYNC_CRUNCH_INTERVAL=${OSM_DATA_SYNC_CRUNCH_INTERVAL}
     depends_on:
       lgd-db:
         condition: service_healthy

linkedgeodata-docker/lgd-nominatim-sync/local.php.dist

@@ -8,7 +8,7 @@
   @define('CONST_Database_DSN', 'pgsql://lgd:lgdpwd@lgd-db:5432/lgd'); // <driver>://<username>:<password>@<host>:<port>/<database>
   @define('CONST_Website_BaseURL', '/');
   @define('CONST_Replication_Url', '${OSM_DATA_SYNC_URL}');
-  @define('CONST_Replication_MaxInterval', '${OSM_DATA_SYNC_UPDATE_INTERVAL}');     // Process each update separately, osmosis cannot merge multiple updates
+  @define('CONST_Replication_MaxInterval', '${OSM_DATA_SYNC_CRUNCH_INTERVAL}');     // Process each update separately, osmosis cannot merge multiple updates
   @define('CONST_Replication_Update_Interval', '${OSM_DATA_SYNC_UPDATE_INTERVAL}');  // How often upstream publishes diffs
   @define('CONST_Replication_Recheck_Interval', '${OSM_DATA_SYNC_RECHECK_INTERVAL}');   // How long to sleep if no update found yet
 ?>

linkedgeodata-docker/lgd-osm-sync/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:16.04
 #FROM mdillon/postgis:9.5
 MAINTAINER Claus Stadler <cstadler@informatik.uni-leipzig.de>
 
-RUN echo "forcerebuild 6"
+RUN echo "forcerebuild 8"
 
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get -y update --fix-missing && apt-get install -y postgresql-9.5-postgis-2.2 wget sudo curl osmosis gettext-base osmctools

linkedgeodata-docker/lgd-osm-sync/configuration.txt.dist

@@ -3,5 +3,5 @@ baseUrl=${OSM_DATA_SYNC_URL}
 
 # Defines the maximum time interval in seconds to download in a single invocation.
 # Setting to 0 disables this feature.
-maxInterval = 3600
+maxInterval=${OSM_DATA_SYNC_CRUNCH_INTERVAL}
 

linkedgeodata-docker/lgd-osm-sync/start.sh

@@ -1,4 +1,5 @@
 #/bin/bash
+set -eu
 
 echo "lgd-osm-sync environment:"
 env | grep -i "osm\|db\|post"
@@ -19,6 +20,11 @@ echo "Retrieved status value from $DB_URL for key [$statusKey] is '$statusVal'"
 
 mkdir -p "$syncDir"
 
+export OSM_DATA_SYNC_UPDATE_INTERVAL=${OSM_DATA_SYNC_UPDATE_INTERVAL:-`lgd-osm-replicate-sequences -u "$OSM_DATA_SYNC_URL" -d`}
+export OSM_DATA_SYNC_RECHECK_INTERVAL=${OSM_DATA_SYNC_RECHECK_INTERVAL:-"$OSM_DATA_SYNC_UPDATE_INTERVAL"}
+
+cat configuration.txt.dist | envsubst > "$syncDir/configuration.txt"
+
 # If the status is empty, then load the data
 if [ -z "$statusVal" ]; then
   #PBF_DATA=http://downloads.linkedgeodata.org/debugging/monaco-170618.osm.pbf
@@ -29,7 +35,6 @@ if [ -z "$statusVal" ]; then
   #curl "https://osm.mazdermind.de/replicate-sequences/?$timestamp" > sync/state.txt
   lgd-osm-replicate-sequences -u "$OSM_DATA_SYNC_URL" -t "$timestamp" > "$syncDir/state.txt"
 
-
 # TODO Fix lgd-createdb to include port
 #  lgd-createdb -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -U "$DB_USER" -W "$DB_PASS" -f "data.osm.bpf" 
   lgd-createdb -h "$DB_HOST" -d "$DB_NAME" -U "$DB_USER" -W "$DB_PASS" -f "$syncDir/data.osm.pbf"
@@ -39,7 +44,6 @@ fi
 
 # NOTE lgd-run-sync uses ./sync/configuration.txt by default
 
-cat configuration.txt.dist | envsubst > "$syncDir/configuration.txt"
 
 # The sync script picks up the environment variables
 # TODO Pass the sync dir to the sync script

linkedgeodata-docker/test.sh

@@ -0,0 +1,2 @@
+test=${test:-30}
+echo "$test"