Datapusher plus support

charts/sddi-ckan/Chart.yaml

@@ -35,6 +35,9 @@ dependencies:
   - name: datapusher
     version: "*"
     condition: datapusher.enabled
+  - name: datapusher-plus
+    version: "*"
+    condition: datapusher-plus.enabled
   - name: certIssuer
     version: "*"
     condition: certIssuer.enabled

charts/sddi-ckan/README.md

@@ -27,6 +27,7 @@ Kubernetes: `>= 1.23.0-0`
 |  | certIssuer | * |
 |  | ckan | * |
 |  | datapusher | * |
+|  | datapusher-plus | * |
 |  | postgis | * |
 |  | redis | * |
 |  | solr | * |
@@ -40,7 +41,8 @@ Kubernetes: `>= 1.23.0-0`
 |-----|------|---------|-------------|
 | cert-manager.enabled | bool | `false` | Enable/disable cert-manager. |
 | certIssuer.enabled | bool | `true` | Enable/disable namespace Issuers for cert-manager. |
-| datapusher.enabled | bool | `true` | Enable/disable Datapusher |
+| datapusher-plus.enabled | bool | `false` | Enable/disable Datapusher-plus. Only one of `datapusher` or `datapusher-plus` should be enabled. |
+| datapusher.enabled | bool | `true` | Enable/disable Datapusher. Only one of `datapusher` or `datapusher-plus` should be enabled. |
 | fullnameOverride | string | `""` | Override fullname |
 | global.datapusher.db.auth.password | string | `"changeMe"` | Jobs database password. If set, this values will overwrite the value in the Datapusher chart. |
 | global.datapusher.db.auth.username | string | `"datapusher"` | Jobs database username. If set, this values will overwrite the value in the Datapusher chart. |

charts/sddi-ckan/charts/ckan/README.md

@@ -52,8 +52,8 @@ A Helm chart for SDDI enabled CKAN.
 | component | string | `"ckan"` | Role of CKAN in this chart |
 | datapusher.apiToken | string | `nil` | Datapusher API token, see [CKAN Datapusher settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#datapusher-settings) |
 | datapusher.callback_url_base | string | `"http://ckan:5000/"` | This should be set to cluster internal ckan service domain. [CKAN DataPusher settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#ckan-datapusher-callback-url-base) |
-| datapusher.formats | string | `"csv xls tsv application/csv"` |  |
-| datapusher.url | string | `"http://datapusher:8000/"` | DataPusher endpoint of CKAN. This should be set to the cluster internal DataPusher service domain. |
+| datapusher.formats | string | `"csv xls xlsx xlsm xlsb tsv tab application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet ods application/vnd.oasis.opendocument.spreadsheet"` |  |
+| datapusher.url | string | `"http://datapusher:8800/"` | DataPusher endpoint of CKAN. This should be set to the cluster internal DataPusher service domain. |
 | datastore.auth.ro.password | string | `"changeMe"` | CKAN datastore read only password, used to build `ckan.datastore.read_url` see [CKAN datastore settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#datastore-settings) |
 | datastore.auth.ro.username | string | `"datastore_ro"` | CKAN datastore read only username, used to build `ckan.datastore.read_url` see [CKAN datastore settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#datastore-settings) |
 | datastore.auth.rw.password | string | `"changeMe"` | CKAN datastore read write password, used to build `ckan.datastore.write_url` see [CKAN datastore settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#datastore-settings) |

charts/sddi-ckan/charts/ckan/values.yaml

@@ -462,9 +462,9 @@ datapusher:
   # -- This should be set to cluster internal ckan service domain.
   # [CKAN DataPusher settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#ckan-datapusher-callback-url-base)
   callback_url_base: http://ckan:5000/
-  formats: "csv xls tsv application/csv"
+  formats: "csv xls xlsx xlsm xlsb tsv tab application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet ods application/vnd.oasis.opendocument.spreadsheet"
   # -- DataPusher endpoint of CKAN. This should be set to the cluster internal DataPusher service domain.
-  url: http://datapusher:8000/
+  url: http://datapusher:8800/
 
 smtp:
   # -- [CKAN SMTP settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#email-settings)

charts/sddi-ckan/charts/datapusher-plus/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/sddi-ckan/charts/datapusher-plus/Chart.yaml

@@ -0,0 +1,18 @@
+apiVersion: v2
+name: datapusher-plus
+description: A Helm chart for CKAN Datapusher-plus.
+type: application
+
+home: https://github.com/tum-gis/sddi-ckan-k8s
+sources:
+  - https://github.com/tum-gis/sddi-ckan-k8s
+  - https://github.com/tum-gis/sddi-ckan-k8s/tree/main/sddi-ckan/charts/charts/datapusher-plus
+  - https://github.com/dathere/datapusher-plus
+
+version: 0.1.0
+appVersion: "0.10.1"
+
+maintainers:
+  - email: b.willenborg@tum.de
+    name: Bruno Willenborg
+    url: https://www.asg.ed.tum.de/en/gis/our-team/staff/bruno-willenborg/

charts/sddi-ckan/charts/datapusher-plus/README.md

@@ -0,0 +1,74 @@
+# datapusher-plus
+
+![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.10.1](https://img.shields.io/badge/AppVersion-0.10.1-informational?style=flat-square)
+
+A Helm chart for CKAN Datapusher-plus.
+
+**Homepage:** <https://github.com/tum-gis/sddi-ckan-k8s>
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Bruno Willenborg | <b.willenborg@tum.de> | <https://www.asg.ed.tum.de/en/gis/our-team/staff/bruno-willenborg/> |
+
+## Source Code
+
+* <https://github.com/tum-gis/sddi-ckan-k8s>
+* <https://github.com/tum-gis/sddi-ckan-k8s/tree/main/sddi-ckan/charts/charts/datapusher-plus>
+* <https://github.com/dathere/datapusher-plus>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | [k8s: Assign pods to nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/) |
+| autoscaling.enabled | bool | `false` | Enable/disable pod autoscaling, if disabled `replicaCount` is used to set number of pods. Check requirements of [HorizontalPodAutoscaling](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). |
+| autoscaling.maxReplicas | int | `5` | Maximum number of replicas |
+| autoscaling.minReplicas | int | `1` | Minimum number of replicas |
+| autoscaling.targetCPUUtilizationPercentage | string | `nil` | [HorizontalPodAutoscaling](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). |
+| autoscaling.targetMemoryUtilizationPercentage | string | `nil` | [HorizontalPodAutoscaling](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). |
+| chunkSize | string | `"16384"` | Size of chunks of the data that is being downloaded in bytes |
+| component | string | `"datapusher"` | Role of CKAN Datapusher in this chart |
+| databaseUri | string | `"sqlite:////tmp/job_store.db"` | SQLAlchemy Database URL for job store, see [Datapusher config](https://github.com/ckan/datapusher#datapusher-configuration) for more. Note: If `db.enabled` this setting is ignored. |
+| datapusherRewriteResources | bool | `true` | Enable or disable (boolean) whether Datapusher should rewrite resources uploaded to CKAN's filestore, since Datapusher takes the CKAN Site URL value for generating the resource URL. Default: False |
+| datapusherRewriteUrl | string | `"http://ckan:5000/"` | Sets the rewrite URL that Datapusher will rewrite resources that are uploaded to CKAN's filestore. Default: http://ckan:5000 |
+| datapusherSslVerify | bool | `false` | Enable or disable (boolean) verification of SSL when trying to get resources. Default: True |
+| datastore.dbname | string | `"datastore"` | Datapusher plus write engine URL dbname |
+| datastore.host | string | `"postgis"` | Datapusher plus write engine URL host |
+| datastore.port | int | `5432` | Datapusher plus write engine URL port |
+| db.auth.password | string | `"changeMe"` | Jobs database password. If set, this values will overwrite the value in the Datapusher chart. Note: This values is overwritten by `global.datapusher.db.auth.password`, if set. |
+| db.auth.username | string | `"datapusher"` | Jobs database username. If set, this values will overwrite the value in the Datapusher chart. Note: This values is overwritten by `global.datapusher.db.auth.username`, if set. |
+| db.dbname | string | `"datapusher_jobs"` | Jobs database name. If set, this values will overwrite the value in the Datapusher chart. Note: This values is overwritten by `global.datapusher.db.dbname`, if set. |
+| db.enabled | bool | `true` | Enable/disable PostgreSQL as database backend for [Datapusher](https://github.com/ckan/datapusher#high-availability-setup). If set to false, `databaseUri` is used. Note: This values is overwritten by `global.datapusher.db.enabled`, if set. |
+| db.host | string | `"postgis"` | Jobs database host. If set, this values will overwrite the value in the Datapusher chart. Note: This values is overwritten by `global.datapusher.db.host`, if set. |
+| db.port | int | `5432` | Jobs database port. If set, this values will overwrite the value in the Datapusher chart. Note: This values is overwritten by `global.datapusher.db.port`, if set. |
+| downloadTimeout | string | `"30"` | Timeout limit of the download request |
+| enabled | bool | `true` | Enable/disable CKAN Datapusher |
+| extraEnv | object | `{}` | Extra environment variables. Values need to be quoted. This can be used to overwrite or extend [CKAN settings](https://docs.ckan.org/en/latest/maintaining/configuration.html#ckan-configuration-file). See [ckanext-envvars](https://github.com/okfn/ckanext-envvars) for variable naming conventions. |
+| fullnameOverride | string | `"datapusher"` | Override fullname |
+| image.pullPolicy | string | `"IfNotPresent"` | [Image pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) |
+| image.repository | string | `"tumgis/ckan-datapusher-plus"` | [Image repository](https://kubernetes.io/docs/concepts/containers/images/) |
+| image.tag | string | `""` | Overrides the image tag whose default is the chart `appVersion`. |
+| imagePullSecrets | list | `[]` | [Image pull secrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) |
+| insertRows | string | `"250"` | Number of rows to take from the data and upload them as chunks to datastore |
+| maxContentLength | string | `"10485760"` | Maximum size of content to be uploaded in bytes. |
+| nameOverride | string | `""` | Override name |
+| nodeSelector | object | `{}` | [k8s: Assign pods to nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/) |
+| podAnnotations | object | `{}` | Additional pod annotations |
+| podSecurityContext | object | `{}` | [k8s: Security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) |
+| replicaCount | int | `1` | Number of replicas. Only used if `autoscaling.enabled = false`. |
+| resources.limits.cpu | string | `"1000m"` | [k8s: Resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) |
+| resources.limits.memory | string | `"500Mi"` | [k8s: Resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) |
+| resources.requests.cpu | string | `"250m"` | [k8s: Resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) |
+| resources.requests.memory | string | `"256Mi"` | [k8s: Resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) |
+| securityContext | object | `{}` | [k8s: Security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) |
+| service.port | int | `8800` | Service port |
+| service.type | string | `"ClusterIP"` | Type of service |
+| serviceAccount.annotations | object | `{}` | Annotations to add to the service account |
+| serviceAccount.create | bool | `false` | Specifies whether a service account should be created |
+| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template |
+| tolerations | list | `[]` | [k8S: Taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) |
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)

charts/sddi-ckan/charts/datapusher-plus/templates/_helpers.tpl

@@ -0,0 +1,90 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "datapusher-plus.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "datapusher-plus.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "datapusher-plus.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "datapusher-plus.labels" -}}
+helm.sh/chart: {{ include "datapusher-plus.chart" . }}
+{{ include "datapusher-plus.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "datapusher-plus.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "datapusher-plus.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "datapusher-plus.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "datapusher-plus.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+SQL alchemy url
+*/}}
+{{- define "datapusher-plus.sqlalchemy.url" -}}
+  {{- if .Values.global.datapusher.db.enabled | default .Values.db.enabled -}}
+    postgresql://
+    {{- .Values.global.datapusher.db.auth.username | default .Values.db.auth.username -}}:
+    {{- .Values.global.datapusher.db.auth.password | default .Values.db.auth.password -}}@
+    {{- .Values.global.datapusher.db.host | default .Values.db.host -}}:
+    {{- .Values.global.datapusher.db.port | default .Values.db.port -}}/
+    {{- .Values.global.datapusher.db.dbname | default .Values.db.dbname -}}
+  {{- else -}}
+    {{ .Values.databaseUri }}
+  {{- end }}
+{{- end }}
+
+{{/*
+Write engine URL
+*/}}
+{{- define "datapusher-plus.writeEngine.url" -}}
+    postgresql://
+    {{- .Values.global.datapusher.db.auth.username | default .Values.db.auth.username -}}:
+    {{- .Values.global.datapusher.db.auth.password | default .Values.db.auth.password -}}@
+    {{- .Values.global.datastore.host | default .Values.datastore.host -}}:
+    {{- .Values.global.datastore.port | default .Values.datastore.port -}}/
+    {{- .Values.global.datastore.dbname | default .Values.datastore.dbname -}}
+{{- end }}

charts/sddi-ckan/charts/datapusher-plus/templates/datapusher-configMap.yml

@@ -0,0 +1,22 @@
+{{- if .Values.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "datapusher-plus.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    {{- include "datapusher-plus.labels" . | nindent 4 }}
+    app.kubernetes.io/part-of: sddi-ckan
+    app.kubernetes.io/component: {{ .Values.component }}
+data:
+  DATAPUSHER_MAX_CONTENT_LENGTH: {{ .Values.maxContentLength | quote }}
+  DATAPUSHER_CHUNK_SIZE: {{ .Values.chunkSize | quote }}
+  DATAPUSHER_CHUNK_INSERT_ROWS: {{ .Values.insertRows | quote }}
+  DATAPUSHER_DOWNLOAD_TIMEOUT: {{ .Values.downloadTimeout | quote }}
+  DATAPUSHER_SSL_VERIFY: {{ .Values.datapusherSslVerify | quote }}
+  DATAPUSHER_REWRITE_RESOURCES: {{ .Values.datapusherRewriteResources | quote }}
+  DATAPUSHER_REWRITE_URL: {{ .Values.datapusherRewriteUrl | quote }}
+  {{- with .Values.extraEnv }}
+    {{- toYaml . | nindent 2 }}
+  {{- end }}
+{{- end -}}