From 51fc02e38400219be805d0a0e311f405a74a2c4e Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Sun, 8 Sep 2024 10:54:24 -0300 Subject: [PATCH 1/3] fix: move ckan configuration to external services --- .../templates/ensemble-manager-config.yaml | 17 ++++++++-------- charts/mint/templates/ui-config.yaml | 13 +++++++++--- charts/mint/values.yaml | 20 ++++++++++++------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/charts/mint/templates/ensemble-manager-config.yaml b/charts/mint/templates/ensemble-manager-config.yaml index 1b73d60..12db7cb 100644 --- a/charts/mint/templates/ensemble-manager-config.yaml +++ b/charts/mint/templates/ensemble-manager-config.yaml @@ -23,17 +23,18 @@ data: {{ .execution_engine | toRawJson | nindent 12}}, {{- end }} - {{- if eq .data_catalog.type "CKAN"}} - "data_catalog_api": {{ .data_catalog.api | quote }}, - "data_catalog_type": {{ .data_catalog.type | quote }}, + {{- end}} + + {{ with .external_services.data_catalog }} + {{- if eq .ckan }} + "data_catalog_api": {{ .ckan.url | quote }}, + "data_catalog_type": {{ .ckan.type | quote }}, "data_catalog_extra": { - "owner_organization_id": {{ .data_catalog.extra.owner_organization_id | quote }}, - "owner_provenance_id": {{ .data_catalog.extra.owner_provenance_id | quote }} + "owner_organization_id": {{ .ckan.extra.owner_organization_id | quote }}, + "owner_provenance_id": {{ .ckan.extra.owner_provenance_id | quote }} }, {{- end}} - {{- end}} - - + {{end}} {{- if eq .Values.components.ensemble_manager.config.data_server.type "S3" }} {{- with .Values.components.ensemble_manager.config }} diff --git a/charts/mint/templates/ui-config.yaml b/charts/mint/templates/ui-config.yaml index fca7b75..cb77ea7 100644 --- a/charts/mint/templates/ui-config.yaml +++ b/charts/mint/templates/ui-config.yaml @@ -85,9 +85,16 @@ data: window.REACT_APP_AIRFLOW_API = "https://airflow.mint.isi.edu/api/v1"; window.REACT_APP_AIRFLOW_DAG_DOWNLOAD_THREAD_ID = "download_thread" - {{ if and .Values.components.ensemble_manager.config.data_catalog (eq .Values.components.ensemble_manager.config.data_catalog.type "CKAN") }} - window.REACT_APP_DATA_CATALOG_TYPE = "CKAN"; - window.REACT_APP_DATA_CATALOG_KEY = {{ .Values.secrets.data_catalog.ckan.api_key | quote }}; + {{ with .Values.external_services }} + {{ if .ckan }} + window.REACT_APP_DATA_CATALOG_TYPE = {{ .ckan.type | quote }}; + window.REACT_APP_DATA_CATALOG_KEY = {{ .ckan.api_key | quote }}; + {{ end }} + {{ end }} + + {{ with .Values.components.data_catalog }} + {{ if eq .enabled false }} + window.REACT_APP_DATA_CATALOG_API = {{.Values.external_services.ckan.url | quote }}; {{ end }} {{ end }} diff --git a/charts/mint/values.yaml b/charts/mint/values.yaml index c040734..3ed48ab 100644 --- a/charts/mint/values.yaml +++ b/charts/mint/values.yaml @@ -337,17 +337,10 @@ components: environment: data_dir: /var/mint config: - data_catalog: - api: "http://localhost:5000" - type: "CKAN" - extra: - owner_organization_id: "21033e7d-484d-4974-9340-8c5755242c1f" - owner_provenance_id: "9ef60317-5da5-4050-8bbc-7d6826fee49f" data_server: type: "S3" region: "ap-south-1" bucket: "mintdata" - graphql: endpoint: "localhost:30003/v1/graphql" enable_ssl: false @@ -457,3 +450,16 @@ autoscaling: nodeSelector: {} tolerations: [] affinity: {} + +external_services: + ckan: + url: "http://localhost:5000" + type: "CKAN" + api_key: "CHANGEME" + extra: + owner_organization_id: "21033e7d-484d-4974-9340-8c5755242c1f" + owner_provenance_id: "9ef60317-5da5-4050-8bbc-7d6826fee49f" + s3: + type: "S3" + region: "ap-south-1" + bucket: "mintdata" From 2a5c4975f2fc245a6d146bd26e2dea9e56c05ef1 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Sun, 8 Sep 2024 11:01:49 -0300 Subject: [PATCH 2/3] move: s3 to external services --- charts/mint/Chart.yaml | 2 +- .../mint/templates/ensemble-manager-config.yaml | 16 +++++++--------- charts/mint/values.yaml | 7 +------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/charts/mint/Chart.yaml b/charts/mint/Chart.yaml index 032420a..a463959 100644 --- a/charts/mint/Chart.yaml +++ b/charts/mint/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 3.3.1-alpha.6 +version: 3.3.1-alpha.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/mint/templates/ensemble-manager-config.yaml b/charts/mint/templates/ensemble-manager-config.yaml index 12db7cb..dbee05b 100644 --- a/charts/mint/templates/ensemble-manager-config.yaml +++ b/charts/mint/templates/ensemble-manager-config.yaml @@ -25,7 +25,7 @@ data: {{- end}} - {{ with .external_services.data_catalog }} + {{ with .Values.external_services.data_catalog }} {{- if eq .ckan }} "data_catalog_api": {{ .ckan.url | quote }}, "data_catalog_type": {{ .ckan.type | quote }}, @@ -36,15 +36,13 @@ data: {{- end}} {{end}} - {{- if eq .Values.components.ensemble_manager.config.data_server.type "S3" }} - {{- with .Values.components.ensemble_manager.config }} - "data_server_type": {{ .data_server.type | quote }}, + {{- if .Values.external_services.s3 }} + "data_server_type": {{ .Values.external_services.s3.type | quote }}, "data_server": { - "region": {{ .data_server.region | quote }}, - "bucket": {{ .data_server.bucket | quote }}, - {{- end}} - "access_key": {{ .Values.secrets.data_server.s3.access_key | quote }}, - "secret_access_key": {{ .Values.secrets.data_server.s3.secret_key | quote }} + "region": {{ .Values.external_services.s3.region | quote }}, + "bucket": {{ .Values.external_services.s3.bucket | quote }}, + "access_key": {{ .Values.secrets.external_services.s3.access_key | quote }}, + "secret_access_key": {{ .Values.secrets.external_services.s3.secret_key | quote }} }, {{- end}} diff --git a/charts/mint/values.yaml b/charts/mint/values.yaml index 3ed48ab..80192aa 100644 --- a/charts/mint/values.yaml +++ b/charts/mint/values.yaml @@ -337,10 +337,6 @@ components: environment: data_dir: /var/mint config: - data_server: - type: "S3" - region: "ap-south-1" - bucket: "mintdata" graphql: endpoint: "localhost:30003/v1/graphql" enable_ssl: false @@ -392,10 +388,9 @@ google: key: AIzaSyAkRnERo4F4dy9AhdrWHAN5vdJWs0vZCgM secrets: - data_catalog: + external_services: ckan: api_key: CHANGEME - data_server: s3: access_key: CHANGEME secret_key: CHANGEME From 10d23274db2f7b22ef9b3962b7d8b1522b725bb6 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Sun, 8 Sep 2024 11:04:10 -0300 Subject: [PATCH 3/3] add: enable/disabled external services --- charts/mint/templates/ensemble-manager-config.yaml | 6 +++--- charts/mint/templates/ui-config.yaml | 2 +- charts/mint/values.yaml | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/charts/mint/templates/ensemble-manager-config.yaml b/charts/mint/templates/ensemble-manager-config.yaml index dbee05b..00f0044 100644 --- a/charts/mint/templates/ensemble-manager-config.yaml +++ b/charts/mint/templates/ensemble-manager-config.yaml @@ -25,8 +25,8 @@ data: {{- end}} - {{ with .Values.external_services.data_catalog }} - {{- if eq .ckan }} + {{ with .Values.external_services}} + {{- if .ckan and eq .ckan.enabled true }} "data_catalog_api": {{ .ckan.url | quote }}, "data_catalog_type": {{ .ckan.type | quote }}, "data_catalog_extra": { @@ -36,7 +36,7 @@ data: {{- end}} {{end}} - {{- if .Values.external_services.s3 }} + {{- if .Values.external_services.s3 and .Values.external_services.s3.enabled }} "data_server_type": {{ .Values.external_services.s3.type | quote }}, "data_server": { "region": {{ .Values.external_services.s3.region | quote }}, diff --git a/charts/mint/templates/ui-config.yaml b/charts/mint/templates/ui-config.yaml index cb77ea7..c8d0067 100644 --- a/charts/mint/templates/ui-config.yaml +++ b/charts/mint/templates/ui-config.yaml @@ -86,7 +86,7 @@ data: window.REACT_APP_AIRFLOW_DAG_DOWNLOAD_THREAD_ID = "download_thread" {{ with .Values.external_services }} - {{ if .ckan }} + {{- if .ckan and eq .ckan.enabled true }} window.REACT_APP_DATA_CATALOG_TYPE = {{ .ckan.type | quote }}; window.REACT_APP_DATA_CATALOG_KEY = {{ .ckan.api_key | quote }}; {{ end }} diff --git a/charts/mint/values.yaml b/charts/mint/values.yaml index 80192aa..c9f26b1 100644 --- a/charts/mint/values.yaml +++ b/charts/mint/values.yaml @@ -448,6 +448,7 @@ affinity: {} external_services: ckan: + enabled: false url: "http://localhost:5000" type: "CKAN" api_key: "CHANGEME" @@ -455,6 +456,7 @@ external_services: owner_organization_id: "21033e7d-484d-4974-9340-8c5755242c1f" owner_provenance_id: "9ef60317-5da5-4050-8bbc-7d6826fee49f" s3: + enabled: false type: "S3" - region: "ap-south-1" - bucket: "mintdata" + region: "" + bucket: ""