fix: osm2pgsql as deployment bugs

Dockerfile

@@ -4,10 +4,10 @@ FROM ubuntu:20.04 as build
 
 ENV DEBIAN_FRONTEND=noninteractive
 ARG OSM2PGSQL_REPOSITORY=https://github.com/MapColonies/osm2pgsql.git
-ARG OSM2PGSQL_COMMIT_SHA=dd8651684baf7d5643dcc8e3f34515992fe60aef
-ARG OSMIUM_TOOL_TAG=v1.13.2
-ARG PROTOZERO_TAG=v1.7.0
-ARG LIBOSMIUM_TAG=v2.17.2
+ARG OSM2PGSQL_COMMIT_SHA=6c68ead630575bc2545cd668c6e6dc7b6d4772f9
+ARG OSMIUM_TOOL_TAG=v1.16.0
+ARG PROTOZERO_TAG=v1.7.1
+ARG LIBOSMIUM_TAG=v2.20.0
 
 RUN apt-get -y update && apt -y install \
   make \
@@ -24,7 +24,12 @@ RUN apt-get -y update && apt -y install \
   liblua5.3-dev \
   pandoc \
   git-core \
-  libboost-program-options-dev
+  libboost-program-options-dev \
+  libopencv-dev \
+  nlohmann-json3-dev \
+  libpotrace-dev \
+  lua5.3 \
+  pyosmium
 
 RUN git clone ${OSM2PGSQL_REPOSITORY} ./osm2pgsql && \
   cd osm2pgsql && \

README.md

@@ -117,9 +117,9 @@ Postgres authentication should be defined by the following env varaibles: `PGHOS
 - `NODE_VERSION` - the version of node. currently supports 14 and 16, defaults to 16
 - `OSM2PGSQL_REPOSITORY` - the repository of osm2pgsql to be built, defaults to [MapColonies forked osm2pgsql](https://github.com/MapColonies/osm2pgsql.git).
 - `OSM2PGSQL_COMMIT_SHA` - the commit SHA of osm2pgsql to be built.
-- `OSMIUM_TOOL_TAG` - the tag version of osm2pgsql to be built, defaults to v1.13.2.
-- `PROTOZERO_TAG` - the tag version of osm2pgsql to be built, defaults to v1.7.0.
-- `LIBOSMIUM_TAG` - the tag version of osm2pgsql to be built, defaults to v2.17.2.
+- `OSMIUM_TOOL_TAG` - the tag version of osm2pgsql to be built, defaults to v1.16.0.
+- `PROTOZERO_TAG` - the tag version of osm2pgsql to be built, defaults to v1.7.1.
+- `LIBOSMIUM_TAG` - the tag version of osm2pgsql to be built, defaults to v2.20.0.
 
 ### Building the container
 

config/custom-environment-variables.json

@@ -42,12 +42,21 @@
       "__name": "OSM2PGSQL_GENERATE_EXPIRE_OUTPUT",
       "__format": "boolean"
     },
+    "schema": "OSM2PGSQL_SCHEMA",
     "middleSchema": "OSM2PGSQL_MIDDLE_SCHEMA",
     "logger": {
       "level": "OSM2PGSQL_LOG_LEVEL",
       "progress": {
         "__name": "OSM2PGSQL_LOG_PROGRESS",
         "__format": "boolean"
+      },
+      "sql": {
+        "__name": "OSM2PGSQL_LOG_SQL",
+        "__format": "boolean"
+      },
+      "sqlData": {
+        "__name": "OSM2PGSQL_LOG_SQL_DATA",
+        "__format": "boolean"
       }
     }
   },

config/default.json

@@ -18,10 +18,13 @@
     "processes": 2,
     "output": "flex",
     "generateExpireOutput": true,
+    "schema": "public",
     "middleSchema": "public",
     "logger": {
       "level": "info",
-      "progress": true
+      "progress": true,
+      "sql": false,
+      "sqlData": false
     }
   },
   "osmium": {

helm/templates/configmap.yaml

@@ -29,9 +29,12 @@ data:
   OSM2PGSQL_PROCESSES: {{ .Values.env.osm2pgsql.processes | quote }}
   OSM2PGSQL_OUTPUT: {{ .Values.env.osm2pgsql.output | quote }}
   OSM2PGSQL_GENERATE_EXPIRE_OUTPUT: {{ .Values.env.osm2pgsql.generateExpireOutput | quote }}
-  OSM2PGSQL_LOG_LEVEL: {{ .Values.env.osm2pgsql.logLevel | quote }}
-  OSM2PGSQL_LOG_PROGRESS: {{ .Values.env.osm2pgsql.logProgress | quote }}
-  OSM2PGSQL_MIDDLE_SCHEMA: {{ .Values.env.osm2pgsql.middleSchema }}
+  OSM2PGSQL_LOG_LEVEL: {{ .Values.env.osm2pgsql.log.Level | quote }}
+  OSM2PGSQL_LOG_PROGRESS: {{ .Values.env.osm2pgsql.log.progress | quote }}
+  OSM2PGSQL_LOG_SQL: {{ .Values.env.osm2pgsql.log.sql | quote }}
+  OSM2PGSQL_LOG_SQL_DATA: {{ .Values.env.osm2pgsql.log.sqlData | quote }}
+  OSM2PGSQL_SCHEMA: {{ .Values.env.osm2pgsql.schema | quote }}
+  OSM2PGSQL_MIDDLE_SCHEMA: {{ .Values.env.osm2pgsql.middleSchema | quote }}
   OSMIUM_VERBOSE: {{ .Values.env.osmium.verbose | quote }}
   OSMIUM_PROGRESS: {{ .Values.env.osmium.progress | quote }}
   {{- with .Values.pgboss }}

helm/templates/cronjob.yaml

@@ -23,6 +23,7 @@ metadata:
 {{- end }}
 spec:
   schedule: {{ .Values.cron.schedule | quote }}
+  startingDeadlineSeconds: {{ .Values.cron.startingDeadlineSeconds }}
   successfulJobsHistoryLimit: {{ .Values.cron.successfulJobsHistoryLimit }}
   failedJobsHistoryLimit: {{ .Values.cron.failedJobsHistoryLimit }}
   concurrencyPolicy: {{ .Values.cron.concurrencyPolicy }}

helm/templates/deployment.yaml

@@ -17,7 +17,7 @@ metadata:
     environment: {{ include "osm2pgsql-wrapper.environment" . }}
     release: {{ $releaseName }}
     {{- include "osm2pgsql-wrapper.labels" . | nindent 4 }}
-{{- with .Values.Annotations }}
+{{- with .Values.annotations }}
   annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}
@@ -42,7 +42,7 @@ spec:
 {{- end }}
     spec:
       containers:
-        - name: {{ $releaseName }}-{{ $chartName }}-job
+        - name: {{ $releaseName }}-{{ $chartName }}-deployment
           {{- with .Values.image }}
           {{- if ne $cloudProviderFlavor "minikube" }}
           image: {{ $cloudProviderDockerRegistryUrl }}/{{ .repository }}:{{ $tag }}

helm/templates/job.yaml

@@ -17,7 +17,7 @@ metadata:
     environment: {{ include "osm2pgsql-wrapper.environment" . }}
     release: {{ $releaseName }}
     {{- include "osm2pgsql-wrapper.labels" . | nindent 4 }}
-{{- with .Values.Annotations }}
+{{- with .Values.annotations }}
   annotations:
 {{ toYaml . | indent 4 }}
 {{- end }}

helm/templates/schema-configmap.yaml

@@ -5,5 +5,5 @@ kind: ConfigMap
 metadata:
   name: {{ .Release.Name }}-{{ $chartName }}-schema-configmap
 data:
-  {{ .Values.cli.append.config.fileName }}: {{ tpl (.Files.Get "config/append-config.json") . | quote }}
+  {{ .Values.cli.append.config.fileName }}: {{ printf "%s" .Values.cli.append.config.path | .Files.Get | quote }}
 {{- end }}

helm/values.yaml

@@ -36,6 +36,7 @@ cron:
   successfulJobsHistoryLimit: 1
   failedJobsHistoryLimit: 3
   concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 200
 
 deployment:
   enabled: false
@@ -66,9 +67,13 @@ env:
     processes: 2
     output: flex
     generateExpireOutput: true
+    schema: public
     middleSchema: public
-    logLevel: info
-    logProgress: true
+    log:
+      level: info
+      progress: true
+      sql: false
+      sqlData: false
   osmium:
     verbose: false
     progress: false

src/cli/commands/append/appendFactory.ts

@@ -34,7 +34,7 @@ export const appendCommandFactory: FactoryFunction<CommandModule<GlobalArguments
       .option('forever', { alias: 'f', describe: 'Process diffs in loop forever without stopping', type: 'boolean' })
       .option('waitTimeSeconds', {
         alias: 't',
-        describe: 'The amount of time to wait between state checks when the local state is up to date',
+        describe: 'The amount of time to wait between state checks when the local state is up to date or when error occurs',
         type: 'number',
       })
       .option('s3Acl', {

src/cli/commands/append/appendManager.ts

@@ -161,6 +161,7 @@ export class AppendManager {
         await this.stateTracker.updateRemoteState();
       } catch (error) {
         await mediator?.updateAction({ status: ActionStatus.FAILED });
+        await setTimeoutPromise(waitTimeSeconds * MILLISECONDS_IN_SECOND);
         continue;
       }
 
@@ -222,7 +223,7 @@ export class AppendManager {
     });
 
     const simplifiedDiffPath = join(DATA_DIR, `${this.stateTracker.nextState}.simplified.${DIFF_FILE_EXTENTION}`);
-    await this.osmCommandRunner.mergeChanges([`${diffPath}`, `--output=${simplifiedDiffPath}`]);
+    await this.osmCommandRunner.mergeChanges([`${diffPath}`, `--output=${simplifiedDiffPath}`, `--overwrite`]);
     return simplifiedDiffPath;
   }
 
@@ -265,7 +266,10 @@ export class AppendManager {
           await this.pushExpiredTilesToQueue(localExpireTilesListPath, entity.geometryKey);
         }
       }
-      await fsPromises.unlink(localExpireTilesListPath);
+
+      if (fs.existsSync(localExpireTilesListPath)) {
+        await fsPromises.unlink(localExpireTilesListPath);
+      }
     });
 
     await Promise.all(uploadPromises);

src/commandRunner/osmCommandRunner.ts

@@ -63,7 +63,14 @@ export class OsmCommandRunner {
     osm2pgsqlArgs.push(`--output=${osm2pgsqlConfig.output}`);
     osm2pgsqlArgs.push(`--log-level=${osm2pgsqlConfig.logger.level}`);
     osm2pgsqlArgs.push(`--log-progress=${osm2pgsqlConfig.logger.progress ? 'true' : 'false'}`);
+    osm2pgsqlArgs.push(`--schema=${osm2pgsqlConfig.schema}`);
     osm2pgsqlArgs.push(`--middle-schema=${osm2pgsqlConfig.middleSchema}`);
+    if (osm2pgsqlConfig.logger.sql) {
+      osm2pgsqlArgs.push('--log-sql');
+    }
+    if (osm2pgsqlConfig.logger.sqlData) {
+      osm2pgsqlArgs.push('--log-sql-data');
+    }
 
     const osmiumConfig = config.get<OsmiumConfig>('osmium');
     const osmiumArgs = this.globalCommandArgs.osmium;

src/common/interfaces.ts

@@ -12,10 +12,13 @@ export interface Osm2pgsqlConfig {
   processes: number;
   output: OutputType;
   expireOutput: boolean;
+  schema: string;
   middleSchema: string;
   logger: {
     level: LogLevel;
     progress: boolean;
+    sql: boolean;
+    sqlData: boolean;
   };
 }