Add Obslytics OCP4 Migration Overlay

- Similar to prod but suspends the ETL pipeline Cron triggers - Verification CronWorkflow is, however, enabled - IDH-161
AICoE · Oct 11, 2021 · 03f5540 · 03f5540
1 parent cb24120
commit 03f5540
Show file tree

Hide file tree

Showing 9 changed files with 379 additions and 0 deletions.
diff --git a/obslytics/overlays/ocp4-migration/backfill-cron-workflow.yaml b/obslytics/overlays/ocp4-migration/backfill-cron-workflow.yaml
@@ -0,0 +1,12 @@
+---
+- op: replace
+  path: /spec/schedule
+  value: 15,45 * * * *
+
+- op: replace
+  path: /spec/workflowSpec/ttlSecondsAfterFinished
+  value: 86400
+
+- op: add
+  path: /spec/suspend
+  value: true  # TODO: remove once ready to migrate
diff --git a/obslytics/overlays/ocp4-migration/dataviz.yaml b/obslytics/overlays/ocp4-migration/dataviz.yaml
@@ -0,0 +1,4 @@
+---
+- op: replace
+  path: /spec/template/spec/containers/0/env/2/value
+  value: data/
diff --git a/obslytics/overlays/ocp4-migration/kustomization.yaml b/obslytics/overlays/ocp4-migration/kustomization.yaml
@@ -0,0 +1,45 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: dh-prod-thanos-extractor
+
+resources:
+  - ../../bases/
+
+patchesJson6902:
+  - path: obslytics-cron-workflow.yaml
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: CronWorkflow
+      name: obslytics-data-exporter-cron-workflow
+  - path: backfill-cron-workflow.yaml
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: CronWorkflow
+      name: obslytics-data-exporter-backfill-cron-workflows
+  - path: verification-cron-workflow.yaml
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: CronWorkflow
+      name: obslytics-data-exporter-verification-cron-workflows
+  - path: triggers.yaml
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name: obslytics-data-exporter-workflow-triggers
+  - path: workflow-templates.yaml
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name: obslytics-data-exporter-workflow-template
+  - path: dataviz.yaml
+    target:
+      group: apps.openshift.io
+      version: v1
+      kind: DeploymentConfig
+      name: obslytics-data-exporter-dataviz
diff --git a/obslytics/overlays/ocp4-migration/obslytics-cron-workflow.yaml b/obslytics/overlays/ocp4-migration/obslytics-cron-workflow.yaml
@@ -0,0 +1,12 @@
+---
+- op: replace
+  path: /spec/schedule
+  value: 0,30 * * * *
+
+- op: replace
+  path: /spec/workflowSpec/ttlSecondsAfterFinished
+  value: 86400
+
+- op: add
+  path: /spec/suspend
+  value: true  # TODO: remove once ready to migrate
diff --git a/obslytics/overlays/ocp4-migration/secrets/obslytics-secret.enc.yaml b/obslytics/overlays/ocp4-migration/secrets/obslytics-secret.enc.yaml
diff --git a/obslytics/overlays/ocp4-migration/secrets/secret-generator.yaml b/obslytics/overlays/ocp4-migration/secrets/secret-generator.yaml
@@ -0,0 +1,6 @@
+apiVersion: viaduct.ai/v1
+kind: ksops
+metadata:
+  name: secret-generator
+files:
+  - ./secrets/obslytics-secret.enc.yaml
diff --git a/obslytics/overlays/ocp4-migration/triggers.yaml b/obslytics/overlays/ocp4-migration/triggers.yaml
@@ -0,0 +1,83 @@
+---
+- op: replace
+  path: /spec/templates/0/steps/0/0/arguments/parameters/0/value
+  value: '[
+    "ceph_cluster_total_bytes",
+    "ceph_cluster_total_used_raw_bytes",
+    "cluster_infrastructure_provider",
+    "cluster_installer",
+    "cluster_version",
+    "cluster:capacity_cpu_cores:sum",
+    "cluster:capacity_memory_bytes:sum",
+    "cluster:node_instance_type_count:sum",
+    "id_version_ebs_account_internal:cluster_subscribed",
+    "instance:etcd_object_counts:sum",
+    "node_role_os_version_machine:cpu_capacity_cores:sum",
+    "subscription_sync_total",
+    "workload:cpu_usage_cores:sum",
+    "workload:memory_usage_bytes:sum",
+    "subscription_labels",
+    "ebs_account_account_type_email_domain_internal",
+    "visual_web_terminal_sessions_total",
+    "acm_managed_cluster_info",
+    "cluster:usage:consumption:rhods:cpu:seconds:rate5m",
+    "csv_succeeded",
+    "rhods_total_users",
+    "rhods_aggregate_availability"
+  ]'
+  # TODO: add the following metrics to this list
+  # "cluster_operator_conditions",  # TODO: times out
+- op: replace
+  path: /spec/templates/1/steps/1/0/withItems
+  value:
+    - ceph_cluster_total_bytes
+    - ceph_cluster_total_used_raw_bytes
+    - cluster_infrastructure_provider
+    - cluster_installer
+    - cluster_version
+    - cluster:capacity_cpu_cores:sum
+    - cluster:capacity_memory_bytes:sum
+    - cluster:node_instance_type_count:sum
+    - id_version_ebs_account_internal:cluster_subscribed
+    - instance:etcd_object_counts:sum
+    - node_role_os_version_machine:cpu_capacity_cores:sum
+    - subscription_sync_total
+    - workload:cpu_usage_cores:sum
+    - workload:memory_usage_bytes:sum
+    - subscription_labels
+    - ebs_account_account_type_email_domain_internal
+    - visual_web_terminal_sessions_total
+    - acm_managed_cluster_info
+    - cluster:usage:consumption:rhods:cpu:seconds:rate5m
+    - csv_succeeded
+    - rhods_total_users
+    - rhods_aggregate_availability
+    # TODO: add the following metrics to this list
+    # - cluster_operator_conditions  # TODO: times out
+- op: replace
+  path: /spec/templates/2/steps/0/0/withItems
+  value:
+    - ceph_cluster_total_bytes
+    - ceph_cluster_total_used_raw_bytes
+    - cluster_infrastructure_provider
+    - cluster_installer
+    - cluster_version
+    - cluster:capacity_cpu_cores:sum
+    - cluster:capacity_memory_bytes:sum
+    - cluster:node_instance_type_count:sum
+    - id_version_ebs_account_internal:cluster_subscribed
+    - instance:etcd_object_counts:sum
+    - node_role_os_version_machine:cpu_capacity_cores:sum
+    - subscription_sync_total
+    - workload:cpu_usage_cores:sum
+    - workload:memory_usage_bytes:sum
+    - subscription_labels
+    - ebs_account_account_type_email_domain_internal
+    - visual_web_terminal_sessions_total
+    - acm_managed_cluster_info
+    - cluster:usage:consumption:rhods:cpu:seconds:rate5m
+    - csv_succeeded
+    - rhods_total_users
+    - rhods_aggregate_availability
+    # TODO: add the following metrics to this list
+    # - cluster_operator_conditions  # TODO: times out
diff --git a/obslytics/overlays/ocp4-migration/verification-cron-workflow.yaml b/obslytics/overlays/ocp4-migration/verification-cron-workflow.yaml
@@ -0,0 +1,12 @@
+---
+- op: replace
+  path: /spec/workflowSpec/ttlSecondsAfterFinished
+  value: 86400
+
+- op: replace
+  path: /spec/workflowSpec/templates/1/steps/0/0/arguments/parameters/0/value
+  value: http://argo-server-dh-prod-thanos-extractor.cloud.datahub.psi.redhat.com
+
+- op: replace
+  path: /spec/workflowSpec/templates/1/steps/0/0/arguments/parameters/3/value
+  value: data-hub-alerts@redhat.com
diff --git a/obslytics/overlays/ocp4-migration/workflow-templates.yaml b/obslytics/overlays/ocp4-migration/workflow-templates.yaml
@@ -0,0 +1,127 @@
+---
+- op: replace
+  path: /spec/templates/0/parallelism
+  value: 2
+
+- op: replace
+  path: /spec/templates/1/dag/tasks/0/arguments/parameters/5/value
+  value: ""
+
+- op: replace
+  path: /spec/templates/2/script/image
+  value: docker-registry.default.svc:5000/dh-prod-thanos-extractor/obslytics-data-exporter:latest
+
+- op: replace
+  path: /spec/templates/2/resources/limits/memory
+  value: 12Gi
+
+- op: replace
+  path: /spec/templates/3/script/image
+  value: docker-registry.default.svc:5000/dh-prod-thanos-extractor/obslytics-pyscripts:latest
+
+- op: replace
+  path: /spec/templates/3/script/source
+  value: |
+    set -x
+    python py/gap_detector.py \
+        --storage_config /etc/obslytics-data-exporter/storage-config.yaml \
+        --prefix data/ \
+        --metric {{inputs.parameters.metric}} \
+        --step {{inputs.parameters.step}} \
+        --delay {{inputs.parameters.delay}} \
+        --from_timestamp {{inputs.parameters.backfill_from_timestamp}} \
+        --earliest_timestamp {{inputs.parameters.min_ts}} \
+        --max_timestamps {{inputs.parameters.backfill_count}} \
+        --output_file /mnt/out/tslist \
+        --verbose \
+        --backfill
+
+- op: replace
+  path: /spec/templates/3/resources/limits/memory
+  value: 12Gi
+
+- op: replace
+  path: /spec/templates/3/resources/requests/memory
+  value: 12Gi
+
+- op: replace
+  path: /spec/templates/4/retryStrategy/limit
+  value: 5
+
+- op: replace
+  path: /spec/templates/4/script/image
+  value: docker-registry.default.svc:5000/dh-prod-thanos-extractor/obslytics-data-exporter:latest
+
+- op: replace
+  path: /spec/templates/4/script/resources/limits/memory
+  value: 12Gi
+
+- op: replace
+  path: /spec/templates/5/retryStrategy/limit
+  value: 1
+
+- op: replace
+  path: /spec/templates/5/script/image
+  value: docker-registry.default.svc:5000/dh-prod-thanos-extractor/obslytics-data-exporter:latest
+
+- op: replace
+  path: /spec/templates/5/script/source
+  value: |
+    set -x
+    export GOGC=30
+
+    echo "Calling Script with Params:"
+    epoch=`date --date="{{inputs.parameters.start_timestamp}}" +"%s"`
+    echo "START       : ${epoch} ({{inputs.parameters.start_timestamp}})"
+    echo "METRIC      : {{inputs.parameters.metric}}"
+    echo "STEP        : {{inputs.parameters.step}}"
+    echo "DELAY       : {{inputs.parameters.delay}}"
+    echo "RESOLUTION  : {{inputs.parameters.resolution}}"
+    echo "PREFIX      : data/"
+
+    # Config params
+    export INPUT_CONFIG_FILE="/etc/obslytics-data-exporter/input-config.yaml"
+    export STORAGE_CONFIG_FILE="/etc/obslytics-data-exporter/storage-config.yaml"
+
+    # Input Params
+    export METRIC_MAX_TIME="${epoch}"
+    export METRIC_STEP="{{inputs.parameters.step}}"
+    export METRIC_DELAY="{{inputs.parameters.delay}}"
+    export METRIC_MATCH="{{inputs.parameters.metric}}"
+
+    # Storage Params
+    export OUTPUT_FILE_TYPE="parquet"
+    export OUTPUT_METRIC_RESOLUTION="{{inputs.parameters.resolution}}"
+    export METRIC_STORAGE_PREFIX="data/"
+
+    # Mode Parameters
+    export VERIFY_DATA_STORED="false"
+    export DRYRUN="{{inputs.parameters.dryrun}}"
+
+    bash ./run.sh
+    echo "Export Complete."
+
+- op: replace
+  path: /spec/templates/5/script/resources/request/memory
+  value: 12Gi
+
+- op: replace
+  path: /spec/templates/5/script/resources/limits/memory
+  value: 12Gi
+
+- op: replace
+  path: /spec/templates/7/script/image
+  value: docker-registry.default.svc:5000/dh-prod-thanos-extractor/obslytics-pyscripts:latest
+
+- op: replace
+  path: /spec/templates/7/script/source
+  value: |
+    set -x
+    python py/verify.py \
+        --config_file /etc/obslytics-data-exporter/storage-config.yaml \
+        --prefix data/ \
+        --metric {{inputs.parameters.metric}} \
+        --step {{inputs.parameters.step}} \
+        --from_timestamp {{inputs.parameters.verification_from_timestamp}} \
+        --max_timestamps {{inputs.parameters.verify_count}} \
+        --delay {{inputs.parameters.delay}} \