fix: Merge environment-level annotations into top level in manifest (m…

…eltano#7331)
BuzzCutNorman · Feb 21, 2023 · 3fda9fe · 3fda9fe
1 parent 85b5f2d
commit 3fda9fe
Show file tree

Hide file tree

Showing 11 changed files with 346 additions and 0 deletions.
diff --git a/docs/example-library/meltano-manifest/ending-meltano.yml b/docs/example-library/meltano-manifest/ending-meltano.yml
@@ -1,5 +1,17 @@
 version: 1
 default_environment: userdev
+annotations:
+  meltano.cloud:
+    key1: value1
+    key2: value2
+  airflow:
+    key3: true
+    key4:
+      key5: {}
+      key6: ["list", "values"]
+    key7:
+    - key8: {}
+    - null
 send_anonymous_usage_stats: false
 project_id: 20f9e32a-be7a-41c7-9847-da9712f40d23
 include_paths:

diff --git a/docs/example-library/meltano-manifest/environments/cicd.meltano.yml b/docs/example-library/meltano-manifest/environments/cicd.meltano.yml
@@ -1,5 +1,18 @@
 environments:
 - name: cicd
+  annotations:
+    meltano.cloud:
+      key1: overwrite-key-1
+      environment-cicd-key1: environment-cicd-value1
+      environment-cicd-key2: environment-cicd-value2
+    airflow:
+      environment-cicd-key3: true
+      environment-cicd-key4:
+        environment-cicd-key5: {}
+        environment-cicd-key6: ["environment", "list", "values"]
+      environment-cicd-key7:
+      - environment-cicd-key8: {}
+      - null
   config:
     plugins:
       extractors:

diff --git a/docs/example-library/meltano-manifest/loaders.meltano.yml b/docs/example-library/meltano-manifest/loaders.meltano.yml
@@ -1,6 +1,18 @@
 plugins:
   loaders:
   - name: target-yaml
+    annotations:
+      meltano.cloud:
+        loader-key1: loader-value1
+        loader-key2: loader-value2
+      airflow:
+        loader-key3: true
+        loader-key4:
+          loader-key5: {}
+          loader-key6: ["loader", "list", "values"]
+        loader-key7:
+        - loader-key8: {}
+        - null
     variant: meltanolabs
     pip_url: git+https://github.com/MeltanoLabs/target-yaml.git@20f37a2ae5a85125fed36a733afab179d0bc8a70
     config:

diff --git a/docs/example-library/meltano-manifest/meltano.yml b/docs/example-library/meltano-manifest/meltano.yml
@@ -1,5 +1,17 @@
 version: 1
 default_environment: userdev
+annotations:
+  meltano.cloud:
+    key1: value1
+    key2: value2
+  airflow:
+    key3: true
+    key4:
+      key5: {}
+      key6: ["list", "values"]
+    key7:
+    - key8: {}
+    - null
 send_anonymous_usage_stats: false
 project_id: 20f9e32a-be7a-41c7-9847-da9712f40d23
 include_paths:

diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.cicd.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.cicd.json
@@ -1,4 +1,43 @@
 {
+  "annotations": {
+    "airflow": {
+      "environment-cicd-key3": true,
+      "environment-cicd-key4": {
+        "environment-cicd-key5": {},
+        "environment-cicd-key6": [
+          "environment",
+          "list",
+          "values"
+        ]
+      },
+      "environment-cicd-key7": [
+        {
+          "environment-cicd-key8": {}
+        },
+        null
+      ],
+      "key3": true,
+      "key4": {
+        "key5": {},
+        "key6": [
+          "list",
+          "values"
+        ]
+      },
+      "key7": [
+        {
+          "key8": {}
+        },
+        null
+      ]
+    },
+    "meltano.cloud": {
+      "environment-cicd-key1": "environment-cicd-value1",
+      "environment-cicd-key2": "environment-cicd-value2",
+      "key1": "overwrite-key-1",
+      "key2": "value2"
+    }
+  },
   "default_environment": "userdev",
   "env": {
     "AIRFLOW_VAR_OPERATOR_TYPE": "bash",
@@ -1400,6 +1439,29 @@
     "files": [],
     "loaders": [
       {
+        "annotations": {
+          "airflow": {
+            "loader-key3": true,
+            "loader-key4": {
+              "loader-key5": {},
+              "loader-key6": [
+                "loader",
+                "list",
+                "values"
+              ]
+            },
+            "loader-key7": [
+              {
+                "loader-key8": {}
+              },
+              null
+            ]
+          },
+          "meltano.cloud": {
+            "loader-key1": "loader-value1",
+            "loader-key2": "loader-value2"
+          }
+        },
         "capabilities": [
           "about",
           "stream-maps"

diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.jigsaw.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.jigsaw.json
@@ -1,4 +1,26 @@
 {
+  "annotations": {
+    "airflow": {
+      "key3": true,
+      "key4": {
+        "key5": {},
+        "key6": [
+          "list",
+          "values"
+        ]
+      },
+      "key7": [
+        {
+          "key8": {}
+        },
+        null
+      ]
+    },
+    "meltano.cloud": {
+      "key1": "value1",
+      "key2": "value2"
+    }
+  },
   "default_environment": "userdev",
   "env": {
     "AIRFLOW_VAR_MELTANO_ENVIRONMENT": "userdev",
@@ -1327,6 +1349,29 @@
     "files": [],
     "loaders": [
       {
+        "annotations": {
+          "airflow": {
+            "loader-key3": true,
+            "loader-key4": {
+              "loader-key5": {},
+              "loader-key6": [
+                "loader",
+                "list",
+                "values"
+              ]
+            },
+            "loader-key7": [
+              {
+                "loader-key8": {}
+              },
+              null
+            ]
+          },
+          "meltano.cloud": {
+            "loader-key1": "loader-value1",
+            "loader-key2": "loader-value2"
+          }
+        },
         "capabilities": [
           "about",
           "stream-maps"

diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.json
@@ -1,4 +1,26 @@
 {
+  "annotations": {
+    "airflow": {
+      "key3": true,
+      "key4": {
+        "key5": {},
+        "key6": [
+          "list",
+          "values"
+        ]
+      },
+      "key7": [
+        {
+          "key8": {}
+        },
+        null
+      ]
+    },
+    "meltano.cloud": {
+      "key1": "value1",
+      "key2": "value2"
+    }
+  },
   "default_environment": "userdev",
   "env": {
     "MELTANO_CLI_LOG_CONFIG": "logging.yaml",
@@ -1318,6 +1340,29 @@
     "files": [],
     "loaders": [
       {
+        "annotations": {
+          "airflow": {
+            "loader-key3": true,
+            "loader-key4": {
+              "loader-key5": {},
+              "loader-key6": [
+                "loader",
+                "list",
+                "values"
+              ]
+            },
+            "loader-key7": [
+              {
+                "loader-key8": {}
+              },
+              null
+            ]
+          },
+          "meltano.cloud": {
+            "loader-key1": "loader-value1",
+            "loader-key2": "loader-value2"
+          }
+        },
         "capabilities": [
           "about",
           "stream-maps"

diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.prod.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.prod.json
@@ -1,4 +1,26 @@
 {
+  "annotations": {
+    "airflow": {
+      "key3": true,
+      "key4": {
+        "key5": {},
+        "key6": [
+          "list",
+          "values"
+        ]
+      },
+      "key7": [
+        {
+          "key8": {}
+        },
+        null
+      ]
+    },
+    "meltano.cloud": {
+      "key1": "value1",
+      "key2": "value2"
+    }
+  },
   "default_environment": "userdev",
   "env": {
     "HUB_METRICS_S3_PATH": "s3://prod-meltano-bucket-01/hub_metrics/",
@@ -1412,6 +1434,29 @@
     "files": [],
     "loaders": [
       {
+        "annotations": {
+          "airflow": {
+            "loader-key3": true,
+            "loader-key4": {
+              "loader-key5": {},
+              "loader-key6": [
+                "loader",
+                "list",
+                "values"
+              ]
+            },
+            "loader-key7": [
+              {
+                "loader-key8": {}
+              },
+              null
+            ]
+          },
+          "meltano.cloud": {
+            "loader-key1": "loader-value1",
+            "loader-key2": "loader-value2"
+          }
+        },
         "capabilities": [
           "about",
           "stream-maps"

diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.staging.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.staging.json
@@ -1,4 +1,26 @@
 {
+  "annotations": {
+    "airflow": {
+      "key3": true,
+      "key4": {
+        "key5": {},
+        "key6": [
+          "list",
+          "values"
+        ]
+      },
+      "key7": [
+        {
+          "key8": {}
+        },
+        null
+      ]
+    },
+    "meltano.cloud": {
+      "key1": "value1",
+      "key2": "value2"
+    }
+  },
   "default_environment": "userdev",
   "env": {
     "HUB_METRICS_S3_PATH": "s3://prod-meltano-bucket-01/hub_metrics/staging/",
@@ -1408,6 +1430,29 @@
     "files": [],
     "loaders": [
       {
+        "annotations": {
+          "airflow": {
+            "loader-key3": true,
+            "loader-key4": {
+              "loader-key5": {},
+              "loader-key6": [
+                "loader",
+                "list",
+                "values"
+              ]
+            },
+            "loader-key7": [
+              {
+                "loader-key8": {}
+              },
+              null
+            ]
+          },
+          "meltano.cloud": {
+            "loader-key1": "loader-value1",
+            "loader-key2": "loader-value2"
+          }
+        },
         "capabilities": [
           "about",
           "stream-maps"