diff --git a/chartpress.yaml b/chartpress.yaml index b88afb62..a7f2e04f 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -14,6 +14,8 @@ charts: valuesPath: dbBackupRestore.image planet-dump: valuesPath: planetDump.image + full-history: + valuesPath: fullHistory.image replication-job: valuesPath: replicationJob.image populate-apidb: diff --git a/images/full-history/Dockerfile b/images/full-history/Dockerfile index a64aa3b2..c2295db2 100644 --- a/images/full-history/Dockerfile +++ b/images/full-history/Dockerfile @@ -1,4 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:latest +FROM developmentseed/osmseed-osm-processor:0.1.0-n487.hbce9469 WORKDIR /mnt/data COPY ./start.sh . -CMD ./start.sh +CMD ./start.sh \ No newline at end of file diff --git a/images/full-history/start.sh b/images/full-history/start.sh index ec67e264..e2040d0c 100755 --- a/images/full-history/start.sh +++ b/images/full-history/start.sh @@ -8,15 +8,17 @@ else echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis fi -# Fixing name for file +# Fixing name for historical file date=$(date '+%y%m%d_%H%M') -fullHistoryFile=history-${date}.osm.bz2 +fullHistoryFile=history-${date}.osh.pbf # In case overwrite the file if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then - fullHistoryFile=history-latest.osm.bz2 + fullHistoryFile=history-latest.osh.pbf fi -# State file + +# State file nname stateFile="state.txt" +osm_tmp_file="osm_tmp.osm" # Creating full history osmosis --read-apidb-change \ @@ -26,21 +28,25 @@ osmosis --read-apidb-change \ password=$POSTGRES_PASSWORD \ validateSchemaVersion=no \ readFullHistory=yes \ - --write-xml-change \ - compressionMethod=bzip2 \ - $fullHistoryFile + --write-xml-change \ + compressionMethod=auto \ + $osm_tmp_file + +# Convert file to PBF file +osmium cat $osm_tmp_file -o $fullHistoryFile +osmium fileinfo $fullHistoryFile # AWS if [ $CLOUDPROVIDER == "aws" ]; then - echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" > $stateFile + echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" >$stateFile # Upload to S3 - aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read - aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read + aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read + aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read fi # Google Storage if [ $CLOUDPROVIDER == "gcp" ]; then - echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" > $stateFile + echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" >$stateFile # Upload to GS gsutil cp -a public-read $fullHistoryFile $GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/$stateFile diff --git a/images/osm-processor/Dockerfile b/images/osm-processor/Dockerfile index 90c8778a..56736568 100644 --- a/images/osm-processor/Dockerfile +++ b/images/osm-processor/Dockerfile @@ -1,7 +1,8 @@ -FROM ubuntu:16.04 +FROM ubuntu:20.04 ENV workdir /mnt/data + RUN apt-get -y update -RUN apt-get -y install \ +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \ build-essential \ libboost-program-options-dev \ libbz2-dev \ @@ -10,14 +11,13 @@ RUN apt-get -y install \ cmake \ pandoc \ git \ - python-pip \ + python3 \ + python3-pip \ curl \ unzip \ wget \ software-properties-common \ - python-software-properties \ libz-dev \ - zlib1g-dev \ gdal-bin \ tar \ bzip2 \ @@ -27,7 +27,7 @@ RUN apt-get -y install \ gradle \ apt-utils -# Install osmosis +# Install osmosis 0.47 RUN git clone https://github.com/openstreetmap/osmosis.git WORKDIR osmosis RUN git checkout 9cfb8a06d9bcc948f34a6c8df31d878903d529fc @@ -38,22 +38,29 @@ RUN ln -s "$PWD"/dist/bin/osmosis /usr/bin/osmosis RUN osmosis --version 2>&1 | grep "Osmosis Version" # Install osmium-tool -RUN git clone https://github.com/mapbox/protozero -RUN cd protozero && git checkout 23d48fd2a441c6e3b2852ff84a0ba398e48f74be && mkdir build && cd build && cmake .. && make && make install -RUN git clone https://github.com/osmcode/libosmium -RUN cd libosmium && git checkout a1f88fe44b01863a1ac84efccff54b98bb2dc886 && mkdir build && cd build && cmake .. && make && make install -RUN git clone https://github.com/osmcode/osmium-tool -RUN cd osmium-tool && git checkout ddbcb44f3ec0c1a8d729e69e3cee40d25f5a00b4 && mkdir build && cd build && cmake .. && make && make install +RUN apt-get -y install \ + libbz2-dev \ + libgd-dev \ + libosmium2-dev \ + libprotozero-dev \ + libsqlite3-dev \ + make \ + jq \ + ca-certificates + +# Other useful packages +RUN apt-get install -y \ + osmium-tool \ + pyosmium \ + rsync \ + tmux \ + zsh + +RUN pip install osmium # Install AWS and GCP cli RUN pip install awscli RUN curl -sSL https://sdk.cloud.google.com | bash RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil -# Install postgresql-client -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list' -RUN apt-get update -RUN apt-get install -y postgresql-client - WORKDIR $workdir \ No newline at end of file diff --git a/images/tiler-server/Dockerfile b/images/tiler-server/Dockerfile index f26b7f02..747a2524 100644 --- a/images/tiler-server/Dockerfile +++ b/images/tiler-server/Dockerfile @@ -28,11 +28,13 @@ RUN apk add --update \ python-dev \ linux-headers \ musl-dev \ - openssl-dev + openssl-dev \ + curl # Install aws-cli and gsutil -RUN pip install awscli \ - && pip install gsutil +RUN pip install awscli +RUN curl -sSL https://sdk.cloud.google.com | bash +RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil RUN pip install mercantile \ && apk del build-deps \ diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 3d698cc7..1a6e847e 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -141,8 +141,29 @@ memcached: idEditor: enabled: false +# ==================================================================================================== +# Variables for full-history container +# ==================================================================================================== fullHistory: - enabled: false + enabled: true + image: + name: '' + tag: '' + nodeSelector: + enabled: true + label_key: nodegroup-type + label_value: ohm-non-tiler + schedule: '* */24 * * *' + env: + OVERWRITE_FHISTORY_FILE: false + resources: + enabled: false + requests: + memory: "14Gi" + cpu: "4" + limits: + memory: "16Gi" + cpu: "4" # ==================================================================================================== # Variables for osm-seed database backup and restore