serverless-ci-dbt.yml

variables:
  DISABLE_FAST_DEPLOYS:
  DAGSTER_CLOUD_URL: $DAGSTER_CLOUD_URL
  DAGSTER_CLOUD_API_TOKEN: $DAGSTER_CLOUD_API_TOKEN
  DAGSTER_DBT_PROJECT_NAME: $DAGSTER_DBT_LOCATION_NAME
  DAGSTER_DBT_PROJECT_DIR: "${CI_PROJECT_DIR}"
  DAGSTER_DBT_PACKAGE_DATA_DIR: "${CI_PROJECT_DIR}/$DAGSTER_DBT_PROJECT_NAME/dbt-project"
  # Python versions 3.8 to 3.12 are supported
  PYTHON_VERSION: '3.10'

deploy-branch:
  stage: deploy
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
  image: ghcr.io/dagster-io/dagster-cloud-action:0.1.43
  script:
    # first create the branch deployment
    - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix)
    - export PR_MESSAGE=$(git log -1 --format='%s')
    - export PR_EMAIL=$(git log -1 --format='%ae')
    - export PR_NAME=$(git log -1 --format='%an')
    - export DEPLOYMENT_NAME=$(dagster-cloud branch-deployment create-or-update
      --url $DAGSTER_CLOUD_URL
      --api-token $DAGSTER_CLOUD_API_TOKEN
      --git-repo-name $CI_PROJECT_NAME
      --branch-name $CI_COMMIT_REF_NAME
      --branch-url "${CI_PROJECT_URL}/-/tree/${CI_COMMIT_REF_NAME}"
      --pull-request-url "${CI_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
      --pull-request-id $CI_MERGE_REQUEST_IID
      --commit-hash $CI_COMMIT_SHORT_SHA
      --timestamp $PR_TIMESTAMP
      --commit-message "${PR_MESSAGE}"
      --author-name "${PR_NAME}"
      --author-email $PR_EMAIL)
    # install dbt package
    - pip install pip --upgrade
    - cd $DAGSTER_DBT_PROJECT_DIR/$DAGSTER_DBT_PROJECT_NAME
    - pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0'
    - pip install pyOpenSSL --upgrade
    - cd -
    - which rsync || ( apt-get update -y && apt-get -y install rsync )
    - rsync -avz --exclude $DAGSTER_DBT_PROJECT_NAME --exclude .git $DAGSTER_DBT_PROJECT_DIR/ $DAGSTER_DBT_PACKAGE_DATA_DIR
    - dbt deps --project-dir $DAGSTER_DBT_PACKAGE_DATA_DIR --profiles-dir $DAGSTER_DBT_PACKAGE_DATA_DIR
    - dbt parse --project-dir $DAGSTER_DBT_PACKAGE_DATA_DIR --profiles-dir $DAGSTER_DBT_PACKAGE_DATA_DIR
    - rm $DAGSTER_DBT_PACKAGE_DATA_DIR/target/partial_parse.msgpack
    # then deploy to that branch
    - /gitlab_action/deploy.py ./dagster_cloud.yaml $DEPLOYMENT_NAME
  environment:
    name: branch/$CI_COMMIT_REF_NAME
    on_stop: close_branch

close_branch:
  stage: deploy
  image: ghcr.io/dagster-io/dagster-cloud-action:0.1.43
  when: manual
  only:
    - merge_requests
  script:
    - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix)
    - dagster-cloud branch-deployment create-or-update
      --url $DAGSTER_CLOUD_URL
      --api-token $DAGSTER_CLOUD_API_TOKEN
      --git-repo-name $CI_PROJECT_NAME
      --branch-name $CI_COMMIT_REF_NAME
      --branch-url "${CI_PROJECT_URL}/-/tree/${CI_COMMIT_REF_NAME}"
      --pull-request-url "${CI_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
      --pull-request-id $CI_MERGE_REQUEST_IID
      --pull-request-status "CLOSED"
      --commit-hash $CI_COMMIT_SHORT_SHA
      --timestamp $PR_TIMESTAMP
  environment:
    name: branch/$CI_COMMIT_REF_NAME
    action: stop

deploy:
  stage: deploy
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  image: ghcr.io/dagster-io/dagster-cloud-action:0.1.43
  script:
    # install dbt package
    - pip install pip --upgrade
    - cd $DAGSTER_DBT_PROJECT_DIR/$DAGSTER_DBT_PROJECT_NAME
    - pip install . --upgrade --upgrade-strategy eager
    - dagster-dbt project prepare-and-package --file $DAGSTER_DBT_PROJECT_NAME/project.py
    # The cli command below can be used to manage syncing the prod manifest to branches if state_path is set on the DbtProject
    # - dagster-cloud ci dagster-dbt project manage-state --file $DAGSTER_DBT_PROJECT_NAME/project.py
    # deploy
    - cd -
    - /gitlab_action/deploy.py ./dagster_cloud.yaml