Update imposm import script

developmentseed · Apr 26, 2024 · c9f71b6 · c9f71b6
1 parent 0730e0d
commit c9f71b6
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 24 deletions.
diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile
@@ -41,6 +41,8 @@ ENV PATH $PATH:/usr/local/go/bin
 WORKDIR /go
 ENV GOPATH /go
 RUN go get github.com/omniscale/imposm3
+WORKDIR $GOPATH/src/github.com/omniscale/imposm3
+RUN git checkout v0.11.1
 RUN go install github.com/omniscale/imposm3/cmd/imposm
 ENV PATH $PATH:$GOPATH/bin
 

diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh
@@ -1,16 +1,15 @@
 #!/bin/bash
-set -e
-
-STATEFILE="state.txt"
-PBFFILE="osm.pbf"
-LIMITFILE="limitFile.geojson"
+set -ex
 
 # directories to keep the imposm's cache for updating the db
 WORKDIR=/mnt/data
 CACHE_DIR=$WORKDIR/cachedir
 DIFF_DIR=$WORKDIR/diff
 IMPOSM3_EXPIRE_DIR=$WORKDIR/imposm3_expire_dir
 
+PBFFILE="${WORKDIR}/osm.pbf"
+STATEFILE="state.txt"
+LIMITFILE="limitFile.geojson"
 # # Setting directory
 # settingDir=/osm
 # Folder to store the imposm expider files in s3 or gs
@@ -33,18 +32,9 @@ mkdir -p "$CACHE_DIR" "$DIFF_DIR" "$IMPOSM3_EXPIRE_DIR"
 function getData() {
     ### Get the PBF file from the cloud provider or public URL
     if [ "$TILER_IMPORT_FROM" == "osm" ]; then
-        wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE"
-    elif [ "$TILER_IMPORT_FROM" == "osmseed" ]; then
-        if [ "$CLOUDPROVIDER" == "aws" ]; then
-            # Get the state.txt file from S3
-            aws s3 cp "$AWS_S3_BUCKET/planet/full-history/$STATEFILE" .
-            PBFCloudPath=$(tail -n +1 "$STATEFILE")
-            aws s3 cp "$PBFCloudPath" "$PBFFILE"
-        elif [ "$CLOUDPROVIDER" == "gcp" ]; then
-            # Get the state.txt file from GS
-            gsutil cp "$GCP_STORAGE_BUCKET/planet/full-history/$STATEFILE" .
-            PBFCloudPath=$(tail -n +1 "$STATEFILE")
-            gsutil cp "$PBFCloudPath" "$PBFFILE"
+        if [ ! -f "$PBFFILE" ]; then
+          echo "$PBFFILE does not exist, downloading..."
+          wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE"
         fi
     fi
 }
@@ -123,26 +113,30 @@ function importData() {
     echo "Execute the missing functions"
     psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_helpers.sql
 
-    echo "Import Natural Earth..."
-    ./scripts/natural_earth.sh
-
-    echo "Import OSM Land..."
-    ./scripts/osm_land.sh
+    if [ "$IMPORT_NATURAL_EARTH" = "true" ]; then
+        echo "Importing Natural Earth..."
+        ./scripts/natural_earth.sh
+    fi
 
+    if [ "$IMPORT_OSM_LAND" = "true" ]; then
+        echo "Import OSM Land..."
+        ./scripts/osm_land.sh
+    fi
+
     echo "Import PBF file..."
     if [ -z "$TILER_IMPORT_LIMIT" ]; then
         imposm import \
             -config $WORKDIR/config.json \
             -read $PBFFILE \
             -write \
-            -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR
+            -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR
     else
         wget $TILER_IMPORT_LIMIT -O $WORKDIR/$LIMITFILE
         imposm import \
             -config $WORKDIR/config.json \
             -read $PBFFILE \
             -write \
-            -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR \
+            -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR \
             -limitto $WORKDIR/$LIMITFILE
     fi
 

diff --git a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml
@@ -66,6 +66,10 @@ spec:
             value: {{ .Values.tilerImposm.env.OVERWRITE_STATE | quote }}
           - name: UPLOAD_EXPIRED_FILES
             value: {{ .Values.tilerImposm.env.UPLOAD_EXPIRED_FILES | default "true" | quote }}
+          - name: IMPORT_NATURAL_EARTH
+            value: {{ .Values.tilerImposm.env.IMPORT_NATURAL_EARTH | default "true" | quote }}
+          - name: IMPORT_OSM_LAND
+            value: {{ .Values.tilerImposm.env.IMPORT_OSM_LAND | default "true" | quote }}
           - name: CLOUDPROVIDER
             value: {{ .Values.cloudProvider }}
           # In case cloudProvider=aws

diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml
@@ -378,6 +378,8 @@ tilerImposm:
     TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf
     TILER_IMPORT_LIMIT: https://gist.githubusercontent.com/Rub21/96bdcac5eb11f0b36ba8d0352ac537f4/raw/2606f2e207d4a0d895897a83efa1efacefd36eb4/monaco.geojson
     UPLOAD_EXPIRED_FILES: true
+    IMPORT_NATURAL_EARTH: true
+    IMPORT_OSM_LAND: true
   persistenceDisk:
     enabled: false
     accessMode: ReadWriteOnce