Update full-history container generator

chartpress.yaml

@@ -14,6 +14,8 @@ charts:
         valuesPath: dbBackupRestore.image
       planet-dump:
         valuesPath: planetDump.image
+      full-history:
+        valuesPath: fullHistory.image
       replication-job:
         valuesPath: replicationJob.image
       populate-apidb:

images/full-history/Dockerfile

@@ -1,4 +1,4 @@
-FROM developmentseed/osmseed-osm-processor:latest
+FROM developmentseed/osmseed-osm-processor:0.1.0-n487.hbce9469
 WORKDIR /mnt/data
 COPY ./start.sh .
-CMD ./start.sh
+CMD ./start.sh

images/full-history/start.sh

@@ -8,15 +8,17 @@ else
 	echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
 fi
 
-# Fixing name for file
+# Fixing name for historical file
 date=$(date '+%y%m%d_%H%M')
-fullHistoryFile=history-${date}.osm.bz2
+fullHistoryFile=history-${date}.osh.pbf
 # In case overwrite the file
 if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then
-	fullHistoryFile=history-latest.osm.bz2
+	fullHistoryFile=history-latest.osh.pbf
 fi
-# State file
+
+# State file nname
 stateFile="state.txt"
+osm_tmp_file="osm_tmp.osm"
 
 # Creating full history
 osmosis --read-apidb-change \
@@ -26,21 +28,25 @@ osmosis --read-apidb-change \
 	password=$POSTGRES_PASSWORD \
 	validateSchemaVersion=no \
 	readFullHistory=yes \
-  	--write-xml-change \
-	compressionMethod=bzip2 \
-	$fullHistoryFile
+	--write-xml-change \
+	compressionMethod=auto \
+	$osm_tmp_file
+
+# Convert file to PBF file
+osmium cat $osm_tmp_file -o $fullHistoryFile
+osmium fileinfo $fullHistoryFile
 
 # AWS
 if [ $CLOUDPROVIDER == "aws" ]; then
-	echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" > $stateFile
+	echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" >$stateFile
 	# Upload to S3
-	aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read 
-	aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read 
+	aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read
+	aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read
 fi
 
 # Google Storage
 if [ $CLOUDPROVIDER == "gcp" ]; then
-	echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" > $stateFile
+	echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" >$stateFile
 	# Upload to GS
 	gsutil cp -a public-read $fullHistoryFile $GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile
 	gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/$stateFile

images/osm-processor/Dockerfile

@@ -1,7 +1,8 @@
-FROM ubuntu:16.04
+FROM ubuntu:20.04
 ENV workdir /mnt/data
+
 RUN apt-get -y update
-RUN apt-get -y install \
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install  \
     build-essential \
     libboost-program-options-dev \
     libbz2-dev \
@@ -10,14 +11,13 @@ RUN apt-get -y install \
     cmake \
     pandoc \
     git \
-    python-pip \
+    python3 \
+    python3-pip \
     curl \
     unzip \
     wget \
     software-properties-common \
-    python-software-properties \
     libz-dev \
-    zlib1g-dev \
     gdal-bin \
     tar \
     bzip2 \
@@ -27,7 +27,7 @@ RUN apt-get -y install \
     gradle \
     apt-utils
 
-# Install osmosis
+# Install osmosis 0.47
 RUN git clone https://github.com/openstreetmap/osmosis.git
 WORKDIR osmosis
 RUN git checkout 9cfb8a06d9bcc948f34a6c8df31d878903d529fc
@@ -38,22 +38,29 @@ RUN ln -s "$PWD"/dist/bin/osmosis /usr/bin/osmosis
 RUN osmosis --version 2>&1 | grep "Osmosis Version"
 
 # Install osmium-tool
-RUN git clone https://github.com/mapbox/protozero
-RUN cd protozero && git checkout 23d48fd2a441c6e3b2852ff84a0ba398e48f74be && mkdir build && cd build && cmake .. && make && make install
-RUN git clone https://github.com/osmcode/libosmium
-RUN cd libosmium && git checkout a1f88fe44b01863a1ac84efccff54b98bb2dc886 && mkdir build && cd build && cmake .. && make && make install
-RUN git clone https://github.com/osmcode/osmium-tool
-RUN cd osmium-tool && git checkout ddbcb44f3ec0c1a8d729e69e3cee40d25f5a00b4 && mkdir build && cd build && cmake .. && make && make install
+RUN apt-get -y install \
+    libbz2-dev \
+    libgd-dev \
+    libosmium2-dev \
+    libprotozero-dev \
+    libsqlite3-dev \
+    make \
+    jq \
+    ca-certificates
+
+# Other useful packages
+RUN apt-get install -y \
+    osmium-tool \
+    pyosmium \
+    rsync \
+    tmux \
+    zsh
+
+RUN pip install osmium
 
 # Install AWS and GCP cli
 RUN pip install awscli
 RUN curl -sSL https://sdk.cloud.google.com | bash
 RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
 
-# Install postgresql-client
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" | tee  /etc/apt/sources.list.d/pgdg.list'
-RUN apt-get update
-RUN apt-get install -y postgresql-client
-
 WORKDIR $workdir

images/tiler-server/Dockerfile

@@ -28,11 +28,13 @@ RUN apk add --update \
     python-dev \
     linux-headers \
     musl-dev \
-    openssl-dev
+    openssl-dev \
+    curl
 
 # Install aws-cli and  gsutil
-RUN pip install awscli \
-    && pip install gsutil
+RUN pip install awscli
+RUN curl -sSL https://sdk.cloud.google.com | bash
+RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
 
 RUN pip install mercantile \
     && apk del build-deps \

osm-seed/values.yaml

@@ -141,8 +141,29 @@ memcached:
 idEditor:
   enabled: false
 
+# ====================================================================================================
+# Variables for full-history container
+# ====================================================================================================
 fullHistory:
-  enabled: false
+  enabled: true
+  image:
+    name: ''
+    tag: ''
+  nodeSelector:
+    enabled: true
+    label_key: nodegroup-type
+    label_value: ohm-non-tiler
+  schedule: '* */24 * * *'
+  env:
+    OVERWRITE_FHISTORY_FILE: false
+  resources:
+    enabled: false
+    requests:
+      memory: "14Gi"
+      cpu: "4"
+    limits:
+      memory: "16Gi"
+      cpu: "4"
 
 # ====================================================================================================
 # Variables for osm-seed database backup and restore