From 30699cea232d28a1b7141891d8851ecbb84162ab Mon Sep 17 00:00:00 2001 From: Claus Stadler Date: Thu, 29 Jun 2017 15:06:31 +0200 Subject: [PATCH] more fixes on the docker --- linkedgeodata-cli/bin/lgd-run-sync.sh | 9 +++++---- linkedgeodata-docker/.env | 3 +++ linkedgeodata-docker/docker-compose.yml | 2 ++ linkedgeodata-docker/lgd-nominatim-sync/local.php.dist | 2 +- linkedgeodata-docker/lgd-osm-sync/Dockerfile | 2 +- linkedgeodata-docker/lgd-osm-sync/configuration.txt.dist | 2 +- linkedgeodata-docker/lgd-osm-sync/start.sh | 8 ++++++-- linkedgeodata-docker/test.sh | 2 ++ 8 files changed, 21 insertions(+), 9 deletions(-) create mode 100755 linkedgeodata-docker/test.sh diff --git a/linkedgeodata-cli/bin/lgd-run-sync.sh b/linkedgeodata-cli/bin/lgd-run-sync.sh index a390e74..956415f 100755 --- a/linkedgeodata-cli/bin/lgd-run-sync.sh +++ b/linkedgeodata-cli/bin/lgd-run-sync.sh @@ -4,8 +4,8 @@ configFileName="$1" #osmosis={osmosis:-osmosis} osmosis="osmosis" -targetPath="sync" -sleepInterval="60" +targetPath=${targetPath:-"sync"} +OSM_DATA_SYNC_RECHECK_INTERVAL=${OSM_DATA_SYNC_RECHECK_INTERVAL:-60} log() { echo "[`date +'%Y-%m-%d %H:%M:%S'`] $$ [$1] $2" ; } @@ -62,8 +62,9 @@ applyChanges while [ 1 ]; do "$osmosis" --read-replication-interval workingDirectory=$targetPath --simplify-change --write-xml-change "$targetPath/diff.osc" applyChanges - echo "Going to sleep for $sleepInterval seconds..." - sleep "$sleepInterval" + # TODO Enhance script to compute time duration to next recheck interval based on timestamp in latest state.txt file + echo "Going to sleep for $OSM_DATA_SYNC_RECHECK_INTERVAL seconds..." + sleep "$OSM_DATA_SYNC_RECHECK_INTERVAL" done diff --git a/linkedgeodata-docker/.env b/linkedgeodata-docker/.env index dee0429..d58aefe 100644 --- a/linkedgeodata-docker/.env +++ b/linkedgeodata-docker/.env @@ -22,7 +22,10 @@ DB_MAINTENANCE_WORK_MEM=256MB OSM_DATA_BASE_URL=http://downloads.linkedgeodata.org/debugging/monaco-170618.osm.pbf OSM_DATA_SYNC_URL=http://download.geofabrik.de/europe/monaco-updates/ +# Crunch interval is the interval for which to merge individual updates into a single larger update +OSM_DATA_SYNC_CRUNCH_INTERVAL=86400 +# These constants should not be needed as they can be obtained automatically by analysing the osm data base url #OSM_DATA_SYNC_RECHECK_INTERVAL=900 #OSM_DATA_SYNC_UPDATE_INTERVAL=3600 diff --git a/linkedgeodata-docker/docker-compose.yml b/linkedgeodata-docker/docker-compose.yml index 917f1e6..41d75c3 100644 --- a/linkedgeodata-docker/docker-compose.yml +++ b/linkedgeodata-docker/docker-compose.yml @@ -36,6 +36,7 @@ services: environment: - OSM_DATA_BASE_URL=${OSM_DATA_BASE_URL} - OSM_DATA_SYNC_URL=${OSM_DATA_SYNC_URL} + - OSM_DATA_SYNC_CRUNCH_INTERVAL=${OSM_DATA_SYNC_CRUNCH_INTERVAL} depends_on: lgd-db: condition: service_healthy @@ -68,6 +69,7 @@ services: environment: - OSM_DATA_BASE_URL=${OSM_DATA_BASE_URL} - OSM_DATA_SYNC_URL=${OSM_DATA_SYNC_URL} + - OSM_DATA_SYNC_CRUNCH_INTERVAL=${OSM_DATA_SYNC_CRUNCH_INTERVAL} depends_on: lgd-db: condition: service_healthy diff --git a/linkedgeodata-docker/lgd-nominatim-sync/local.php.dist b/linkedgeodata-docker/lgd-nominatim-sync/local.php.dist index 3ff1ed1..9cbae8d 100644 --- a/linkedgeodata-docker/lgd-nominatim-sync/local.php.dist +++ b/linkedgeodata-docker/lgd-nominatim-sync/local.php.dist @@ -8,7 +8,7 @@ @define('CONST_Database_DSN', 'pgsql://lgd:lgdpwd@lgd-db:5432/lgd'); // ://:@:/ @define('CONST_Website_BaseURL', '/'); @define('CONST_Replication_Url', '${OSM_DATA_SYNC_URL}'); - @define('CONST_Replication_MaxInterval', '${OSM_DATA_SYNC_UPDATE_INTERVAL}'); // Process each update separately, osmosis cannot merge multiple updates + @define('CONST_Replication_MaxInterval', '${OSM_DATA_SYNC_CRUNCH_INTERVAL}'); // Process each update separately, osmosis cannot merge multiple updates @define('CONST_Replication_Update_Interval', '${OSM_DATA_SYNC_UPDATE_INTERVAL}'); // How often upstream publishes diffs @define('CONST_Replication_Recheck_Interval', '${OSM_DATA_SYNC_RECHECK_INTERVAL}'); // How long to sleep if no update found yet ?> diff --git a/linkedgeodata-docker/lgd-osm-sync/Dockerfile b/linkedgeodata-docker/lgd-osm-sync/Dockerfile index 1ab974f..24e7371 100644 --- a/linkedgeodata-docker/lgd-osm-sync/Dockerfile +++ b/linkedgeodata-docker/lgd-osm-sync/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:16.04 #FROM mdillon/postgis:9.5 MAINTAINER Claus Stadler -RUN echo "forcerebuild 6" +RUN echo "forcerebuild 8" ENV DEBIAN_FRONTEND noninteractive RUN apt-get -y update --fix-missing && apt-get install -y postgresql-9.5-postgis-2.2 wget sudo curl osmosis gettext-base osmctools diff --git a/linkedgeodata-docker/lgd-osm-sync/configuration.txt.dist b/linkedgeodata-docker/lgd-osm-sync/configuration.txt.dist index 7178977..a76a27f 100644 --- a/linkedgeodata-docker/lgd-osm-sync/configuration.txt.dist +++ b/linkedgeodata-docker/lgd-osm-sync/configuration.txt.dist @@ -3,5 +3,5 @@ baseUrl=${OSM_DATA_SYNC_URL} # Defines the maximum time interval in seconds to download in a single invocation. # Setting to 0 disables this feature. -maxInterval = 3600 +maxInterval=${OSM_DATA_SYNC_CRUNCH_INTERVAL} diff --git a/linkedgeodata-docker/lgd-osm-sync/start.sh b/linkedgeodata-docker/lgd-osm-sync/start.sh index e915523..4acf465 100644 --- a/linkedgeodata-docker/lgd-osm-sync/start.sh +++ b/linkedgeodata-docker/lgd-osm-sync/start.sh @@ -1,4 +1,5 @@ #/bin/bash +set -eu echo "lgd-osm-sync environment:" env | grep -i "osm\|db\|post" @@ -19,6 +20,11 @@ echo "Retrieved status value from $DB_URL for key [$statusKey] is '$statusVal'" mkdir -p "$syncDir" +export OSM_DATA_SYNC_UPDATE_INTERVAL=${OSM_DATA_SYNC_UPDATE_INTERVAL:-`lgd-osm-replicate-sequences -u "$OSM_DATA_SYNC_URL" -d`} +export OSM_DATA_SYNC_RECHECK_INTERVAL=${OSM_DATA_SYNC_RECHECK_INTERVAL:-"$OSM_DATA_SYNC_UPDATE_INTERVAL"} + +cat configuration.txt.dist | envsubst > "$syncDir/configuration.txt" + # If the status is empty, then load the data if [ -z "$statusVal" ]; then #PBF_DATA=http://downloads.linkedgeodata.org/debugging/monaco-170618.osm.pbf @@ -29,7 +35,6 @@ if [ -z "$statusVal" ]; then #curl "https://osm.mazdermind.de/replicate-sequences/?$timestamp" > sync/state.txt lgd-osm-replicate-sequences -u "$OSM_DATA_SYNC_URL" -t "$timestamp" > "$syncDir/state.txt" - # TODO Fix lgd-createdb to include port # lgd-createdb -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -U "$DB_USER" -W "$DB_PASS" -f "data.osm.bpf" lgd-createdb -h "$DB_HOST" -d "$DB_NAME" -U "$DB_USER" -W "$DB_PASS" -f "$syncDir/data.osm.pbf" @@ -39,7 +44,6 @@ fi # NOTE lgd-run-sync uses ./sync/configuration.txt by default -cat configuration.txt.dist | envsubst > "$syncDir/configuration.txt" # The sync script picks up the environment variables # TODO Pass the sync dir to the sync script diff --git a/linkedgeodata-docker/test.sh b/linkedgeodata-docker/test.sh new file mode 100755 index 0000000..e5a482d --- /dev/null +++ b/linkedgeodata-docker/test.sh @@ -0,0 +1,2 @@ +test=${test:-30} +echo "$test"