userdev.meltano.yml

environments:
- name: userdev
  config:
    plugins:
      extractors:
      - name: tap-cloudwatch
        config:
          start_date: "2023-01-16"
      - name: tap-slack-public
        config:
          start_date: '2022-09-01'
      - name: tap-google-analytics
        config:
          start_date: '2022-02-20'
      - name: tap-gitlab
        config:
          groups: meltano
          start_date: '2020-01-01T00:00:00Z'
      - name: tap-github
        config:
          start_date: '2021-05-01'
      - name: tap-snowflake
        config:
          dbname: USERDEV_PROD
          user: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
      - name: tap-snowflake-metrics-legacy
        config:
          tables: USERDEV_PROD.${USER_PREFIX}_MELTANO_HUB.FACT_HUB_METRICS
      - name: tap-snowflake-metrics
        config:
          tables: USERDEV_PROD.${USER_PREFIX}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
      - name: tap-snowflake-audit
        config:
          tables: USERDEV_PROD.${USER_PREFIX}_MELTANO_HUB.HUB_METRICS_AUDIT
      - name: tap-snowflake-singer-activity
        config:
          tables: USERDEV_PROD.${USER_PREFIX}_SLACK_NOTIFICATIONS.SLACK_ALERTS
      loaders:
      - name: target-snowflake
        config:
          dbname: USERDEV_RAW
          user: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
          default_target_schema: ${USER_PREFIX}_${MELTANO_EXTRACT__LOAD_SCHEMA}
      - name: target-apprise
        config:
          stream_maps:
            USERDEV_PROD-PNADOLNY_SLACK_NOTIFICATIONS-SLACK_ALERTS:
              TITLE: null
              title: TITLE
              BODY: null
              body: BODY
      utilities:
      - name: dbt-snowflake
        config:
          user: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
          skip_pre_invoke: True
          database: USERDEV_PROD
          database_prep: USERDEV_PREP
          # database_raw: RAW
          # Uncomment for reading a custom RAW schema
          database_raw: USERDEV_RAW
          source_schema_prefix: ${USER_PREFIX}_
          target_schema_prefix: ${USER_PREFIX}_
      - name: sqlfluff
        config:
          user: ${USER_PREFIX}
      - name: great_expectations
        config:
          prod_database: USERDEV_PROD
          raw_database: USERDEV_RAW
          username: ${USER_PREFIX}
          role: ${USER_PREFIX}
          warehouse: CORE
  env:
    USER_PREFIX: PNADOLNY
    HUB_METRICS_S3_PATH: s3://devtest-meltano-bucket-01/hub_metrics/
    SUPERSET_API_URL: http://localhost:8088
    SUPERSET_USER: admin
    SUPERSET_PASS: admin
    # https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html
    AIRFLOW__CORE__PLUGINS_FOLDER: $MELTANO_PROJECT_ROOT/orchestrate/plugins_local
    AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: '30'
    AIRFLOW_VAR_MELTANO_ENVIRONMENT: userdev
    AIRFLOW_VAR_OPERATOR_TYPE: bash
    # Secrets via KMS
    KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem
    KMS_DOTENV_PATH: .env
    KMS_SECTRETS_PATH: secrets.yml