ci: Run manifest integration test in CI (meltano#7314)

BuzzCutNorman · Feb 15, 2023 · 313d5f0 · 313d5f0
1 parent 5c97e3b
commit 313d5f0
Show file tree

Hide file tree

Showing 11 changed files with 720 additions and 352 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -37,6 +37,7 @@ jobs:
         - { integration_test: "meltano-state-s3", needs_postgres: false}
         - { integration_test: "meltano-state-local", needs_postgres: false}
         - { integration_test: "meltano-config", needs_postgres: false}
+        - { integration_test: "meltano-manifest", needs_postgres: false}
       fail-fast: false
 
     runs-on: ubuntu-latest

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -25,7 +25,7 @@ repos:
     rev: v4.4.0
     hooks:
       - id: trailing-whitespace
-        exclude: (cli_messages\.py)
+        exclude: ^src/meltano/core/cli_messages\.py$
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-json

diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.cicd.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.cicd.json
diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.jigsaw.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.jigsaw.json
diff --git a/integration/meltano-manifest/expected-manifests/meltano-manifest.json b/integration/meltano-manifest/expected-manifests/meltano-manifest.json
@@ -5,7 +5,7 @@
     "MELTANO_CLI_LOG_LEVEL": "info",
     "MELTANO_DATABASE_MAX_RETRIES": "3",
     "MELTANO_DATABASE_RETRY_TIMEOUT": "5",
-    "MELTANO_DATABASE_URI": "sqlite:////home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/meltano.db",
+    "MELTANO_DATABASE_URI": "sqlite:///${MELTANO_PROJECT_ROOT}/.meltano/meltano.db",
     "MELTANO_DEFAULT_ENVIRONMENT": "userdev",
     "MELTANO_DISCOVERY_URL": "https://discovery.meltano.com/discovery.yml",
     "MELTANO_ELT_BUFFER_SIZE": "10485760",
@@ -41,15 +41,6 @@
     "MELTANO_UI_SESSION_COOKIE_SECURE": "false",
     "MELTANO_UI_WORKERS": "4"
   },
-  "include_paths": [
-    "environments/*.meltano.yml",
-    "extractors.meltano.yml",
-    "mappers.meltano.yml",
-    "loaders.meltano.yml",
-    "orchestrators.meltano.yml",
-    "transformers.meltano.yml",
-    "utilities.meltano.yml"
-  ],
   "jobs": [
     {
       "name": "slack_notifications",
@@ -1879,39 +1870,39 @@
         },
         "docs": "https://hub.meltano.com/utilities/airflow--apache",
         "env": {
-          "AIRFLOW_CONFIG": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/airflow.cfg",
+          "AIRFLOW_CONFIG": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/airflow.cfg",
           "AIRFLOW_CORE_DAGS_ARE_PAUSED_AT_CREATION": "False",
-          "AIRFLOW_CORE_DAGS_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/dags",
+          "AIRFLOW_CORE_DAGS_FOLDER": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/dags",
           "AIRFLOW_CORE_LOAD_EXAMPLES": "False",
-          "AIRFLOW_CORE_PLUGINS_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/plugins",
-          "AIRFLOW_DATABASE_SQL_ALCHEMY_CONN": "sqlite:////home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/airflow.db",
-          "AIRFLOW_EXTENSION_AIRFLOW_CONFIG": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/airflow.cfg",
-          "AIRFLOW_EXTENSION_AIRFLOW_HOME": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow",
-          "AIRFLOW_HOME": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow",
-          "AIRFLOW_LOGGING_BASE_LOG_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs",
-          "AIRFLOW_LOGGING_DAG_PROCESSOR_MANAGER_LOG_LOCATION": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs/dag_processor_manager/dag_processor_manager.log",
-          "AIRFLOW_SCHEDULER_CHILD_PROCESS_LOG_DIRECTORY": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs/scheduler",
+          "AIRFLOW_CORE_PLUGINS_FOLDER": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/plugins",
+          "AIRFLOW_DATABASE_SQL_ALCHEMY_CONN": "sqlite:///${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/airflow.db",
+          "AIRFLOW_EXTENSION_AIRFLOW_CONFIG": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/airflow.cfg",
+          "AIRFLOW_EXTENSION_AIRFLOW_HOME": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow",
+          "AIRFLOW_HOME": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow",
+          "AIRFLOW_LOGGING_BASE_LOG_FOLDER": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs",
+          "AIRFLOW_LOGGING_DAG_PROCESSOR_MANAGER_LOG_LOCATION": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs/dag_processor_manager/dag_processor_manager.log",
+          "AIRFLOW_SCHEDULER_CHILD_PROCESS_LOG_DIRECTORY": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs/scheduler",
           "AIRFLOW_WEBSERVER_WEB_SERVER_PORT": "8080",
           "AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION": "False",
-          "AIRFLOW__CORE__DAGS_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/dags",
+          "AIRFLOW__CORE__DAGS_FOLDER": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/dags",
           "AIRFLOW__CORE__LOAD_EXAMPLES": "False",
-          "AIRFLOW__CORE__PLUGINS_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/plugins",
-          "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN": "sqlite:////home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/airflow.db",
+          "AIRFLOW__CORE__PLUGINS_FOLDER": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/plugins",
+          "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN": "sqlite:///${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/airflow.db",
           "AIRFLOW__EXT_REPO": "https://github.com/meltano/airflow-ext",
-          "AIRFLOW__LOGGING__BASE_LOG_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs",
-          "AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs/dag_processor_manager/dag_processor_manager.log",
-          "AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs/scheduler",
+          "AIRFLOW__LOGGING__BASE_LOG_FOLDER": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs",
+          "AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs/dag_processor_manager/dag_processor_manager.log",
+          "AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs/scheduler",
           "AIRFLOW__WEBSERVER__WEB_SERVER_PORT": "8080",
           "MELTANO_UTILIZE_CORE_DAGS_ARE_PAUSED_AT_CREATION": "False",
-          "MELTANO_UTILIZE_CORE_DAGS_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/dags",
+          "MELTANO_UTILIZE_CORE_DAGS_FOLDER": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/dags",
           "MELTANO_UTILIZE_CORE_LOAD_EXAMPLES": "False",
-          "MELTANO_UTILIZE_CORE_PLUGINS_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/plugins",
-          "MELTANO_UTILIZE_DATABASE_SQL_ALCHEMY_CONN": "sqlite:////home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/airflow.db",
-          "MELTANO_UTILIZE_EXTENSION_AIRFLOW_CONFIG": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow/airflow.cfg",
-          "MELTANO_UTILIZE_EXTENSION_AIRFLOW_HOME": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/orchestrate/airflow",
-          "MELTANO_UTILIZE_LOGGING_BASE_LOG_FOLDER": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs",
-          "MELTANO_UTILIZE_LOGGING_DAG_PROCESSOR_MANAGER_LOG_LOCATION": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs/dag_processor_manager/dag_processor_manager.log",
-          "MELTANO_UTILIZE_SCHEDULER_CHILD_PROCESS_LOG_DIRECTORY": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/airflow/logs/scheduler",
+          "MELTANO_UTILIZE_CORE_PLUGINS_FOLDER": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/plugins",
+          "MELTANO_UTILIZE_DATABASE_SQL_ALCHEMY_CONN": "sqlite:///${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/airflow.db",
+          "MELTANO_UTILIZE_EXTENSION_AIRFLOW_CONFIG": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow/airflow.cfg",
+          "MELTANO_UTILIZE_EXTENSION_AIRFLOW_HOME": "${MELTANO_PROJECT_ROOT}/orchestrate/airflow",
+          "MELTANO_UTILIZE_LOGGING_BASE_LOG_FOLDER": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs",
+          "MELTANO_UTILIZE_LOGGING_DAG_PROCESSOR_MANAGER_LOG_LOCATION": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs/dag_processor_manager/dag_processor_manager.log",
+          "MELTANO_UTILIZE_SCHEDULER_CHILD_PROCESS_LOG_DIRECTORY": "${MELTANO_PROJECT_ROOT}/.meltano/utilities/airflow/logs/scheduler",
           "MELTANO_UTILIZE_WEBSERVER_WEB_SERVER_PORT": "8080",
           "MELTANO_UTILIZE__EXT_REPO": "https://github.com/meltano/airflow-ext"
         },
@@ -2098,20 +2089,20 @@
         "env": {
           "DBT_EXT_SKIP_PRE_INVOKE": "false",
           "DBT_EXT_TYPE": "snowflake",
-          "DBT_PROFILES_DIR": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/transform/profiles/snowflake",
+          "DBT_PROFILES_DIR": "${MELTANO_PROJECT_ROOT}/transform/profiles/snowflake",
           "DBT_SNOWFLAKE_ACCOUNT": "epa06486",
           "DBT_SNOWFLAKE_DATABASE_PREP": "PREP",
           "DBT_SNOWFLAKE_DATABASE_RAW": "RAW",
-          "DBT_SNOWFLAKE_PROFILES_DIR": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/transform/profiles/snowflake",
-          "DBT_SNOWFLAKE_PROJECT_DIR": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/transform",
+          "DBT_SNOWFLAKE_PROFILES_DIR": "${MELTANO_PROJECT_ROOT}/transform/profiles/snowflake",
+          "DBT_SNOWFLAKE_PROJECT_DIR": "${MELTANO_PROJECT_ROOT}/transform",
           "DBT_SNOWFLAKE_SKIP_PRE_INVOKE": "false",
           "DBT_SNOWFLAKE_TYPE": "snowflake",
           "DBT_SNOWFLAKE__EXT_REPO": "https://github.com/meltano/dbt-ext",
           "MELTANO_UTILIZE_ACCOUNT": "epa06486",
           "MELTANO_UTILIZE_DATABASE_PREP": "PREP",
           "MELTANO_UTILIZE_DATABASE_RAW": "RAW",
-          "MELTANO_UTILIZE_PROFILES_DIR": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/transform/profiles/snowflake",
-          "MELTANO_UTILIZE_PROJECT_DIR": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/transform",
+          "MELTANO_UTILIZE_PROFILES_DIR": "${MELTANO_PROJECT_ROOT}/transform/profiles/snowflake",
+          "MELTANO_UTILIZE_PROJECT_DIR": "${MELTANO_PROJECT_ROOT}/transform",
           "MELTANO_UTILIZE_SKIP_PRE_INVOKE": "false",
           "MELTANO_UTILIZE_TYPE": "snowflake",
           "MELTANO_UTILIZE__EXT_REPO": "https://github.com/meltano/dbt-ext"
@@ -2311,13 +2302,13 @@
         "docs": "https://docs.meltano.com/guide/analysis",
         "env": {
           "MELTANO_UTILIZE_SECRET_KEY": "thisisnotapropersecretkey",
-          "MELTANO_UTILIZE_SQLALCHEMY_DATABASE_URI": "sqlite:////home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/superset/superset.db",
+          "MELTANO_UTILIZE_SQLALCHEMY_DATABASE_URI": "sqlite:///${MELTANO_PROJECT_ROOT}/.meltano/utilities/superset/superset.db",
           "MELTANO_UTILIZE_UI_BIND_HOST": "0.0.0.0",
           "MELTANO_UTILIZE_UI_PORT": "8088",
           "MELTANO_UTILIZE_UI_TIMEOUT": "60",
           "MELTANO_UTILIZE_UI_WORKERS": "4",
           "SUPERSET_SECRET_KEY": "thisisnotapropersecretkey",
-          "SUPERSET_SQLALCHEMY_DATABASE_URI": "sqlite:////home/will/meltano/meltano/docs/example-library/meltano-manifest/.meltano/utilities/superset/superset.db",
+          "SUPERSET_SQLALCHEMY_DATABASE_URI": "sqlite:///${MELTANO_PROJECT_ROOT}/.meltano/utilities/superset/superset.db",
           "SUPERSET_UI_BIND_HOST": "0.0.0.0",
           "SUPERSET_UI_PORT": "8088",
           "SUPERSET_UI_TIMEOUT": "60",
@@ -2381,10 +2372,10 @@
         },
         "docs": "https://hub.meltano.com/utilities/great_expectations--great-expectations",
         "env": {
-          "GE_HOME": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/utilities/great_expectations",
-          "GREAT_EXPECTATIONS_GE_HOME": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/utilities/great_expectations",
+          "GE_HOME": "${MELTANO_PROJECT_ROOT}/utilities/great_expectations",
+          "GREAT_EXPECTATIONS_GE_HOME": "${MELTANO_PROJECT_ROOT}/utilities/great_expectations",
           "GREAT_EXPECTATIONS_HOST": "epa06486",
-          "MELTANO_UTILIZE_GE_HOME": "/home/will/meltano/meltano/docs/example-library/meltano-manifest/utilities/great_expectations",
+          "MELTANO_UTILIZE_GE_HOME": "${MELTANO_PROJECT_ROOT}/utilities/great_expectations",
           "MELTANO_UTILIZE_HOST": "epa06486"
         },
         "executable": "great_expectations",