diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35382d39..9051df50 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,18 +77,39 @@ View changes made by the release script # see also dagster-cloud-action-test ``` +# Step 5b. Iterate + +If you make any changes and rebuild the rc, you need to retag and push the release tag: + +``` +python scripts/release.py create-rc 0.1.22 0.1.21 + +# the only autogenerated change should be to the pex file because changes to the yamls have already committed + +# -f to force since the tag exists +git tag -f -a v0.1.22 +git push -f origin v0.1.22 + +# test again, fix and repeat +``` + # Step 6. Promote Most users point at a dot version tag for the GitHub Action, e.g. `@v0.1` and `@pex-v0.1`. If you are releasing a fix or non-breaking feature, you want to move this tag so existing users get access to your changes. +> Due an unfortunate bug, the ci-summary step in many user's workflow is pinned to v0.1.27, so we need v0.1.27 to follow v0.1 as well. +> See https://github.com/dagster-io/dagster-cloud-hybrid-quickstart/commit/2149359dd1076f3bcd6f652dc00cb74ac9c81636 + ``` # use '-f' to force move the tag, since these tags already exist git tag -f v0.1 v0.1.22 git tag -f pex-v0.1 v0.1.22 +git tag -f v0.1.27 v0.1 # needs a force push to move remote tags git push -f origin v0.1 git push -f origin pex-v0.1 +git push -f origin v0.1.27 ``` Note: both `v0.1` and `pex-v0.1` tags should be kept in sync. diff --git a/actions/build_deploy_python_executable/action.yml b/actions/build_deploy_python_executable/action.yml index 78acbbdf..8391dba7 100644 --- a/actions/build_deploy_python_executable/action.yml +++ b/actions/build_deploy_python_executable/action.yml @@ -48,16 +48,20 @@ runs: shell: bash - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8" - if: ${{ inputs.python_version != '3.8' }} + id: custom-python-version name: Set up Python ${{ inputs.python_version }} for target - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} - run: pip install setuptools + + - if: ${{ inputs.python_version != '3.8' }} + run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools + shell: bash - run: > echo SOURCE_DIRECTORY=$(dirname ${{ inputs.dagster_cloud_file }}) >> $GITHUB_ENV diff --git a/actions/utils/copy_template/action.yml b/actions/utils/copy_template/action.yml index 0d5a248c..b9efb14f 100644 --- a/actions/utils/copy_template/action.yml +++ b/actions/utils/copy_template/action.yml @@ -12,5 +12,5 @@ inputs: description: "A string of the base image name for the deployed code location image." runs: using: "docker" - image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.38" + image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.39" entrypoint: "/copy_template.sh" diff --git a/actions/utils/deploy/action.yml b/actions/utils/deploy/action.yml index 77f1a01d..1cd434e8 100644 --- a/actions/utils/deploy/action.yml +++ b/actions/utils/deploy/action.yml @@ -39,7 +39,7 @@ outputs: description: "The Cloud deployment associated with this branch." runs: using: "docker" - image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.38" + image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.39" entrypoint: "/deploy.sh" args: - ${{ inputs.pr }} diff --git a/actions/utils/get_branch_deployment/action.yml b/actions/utils/get_branch_deployment/action.yml index a68556ae..4d4dd058 100644 --- a/actions/utils/get_branch_deployment/action.yml +++ b/actions/utils/get_branch_deployment/action.yml @@ -15,5 +15,5 @@ outputs: description: "The Cloud deployment associated with this branch." runs: using: "docker" - image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.38" + image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.39" entrypoint: "/get_branch_deployment.sh" diff --git a/actions/utils/notify/action.yml b/actions/utils/notify/action.yml index aaa25411..2ffefd7e 100644 --- a/actions/utils/notify/action.yml +++ b/actions/utils/notify/action.yml @@ -30,7 +30,7 @@ inputs: runs: using: "docker" - image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.38" + image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.39" entrypoint: "/notify.sh" args: - ${{ inputs.pr }} diff --git a/actions/utils/registry_info/action.yml b/actions/utils/registry_info/action.yml index 3261ba59..e98e38d5 100644 --- a/actions/utils/registry_info/action.yml +++ b/actions/utils/registry_info/action.yml @@ -12,5 +12,5 @@ inputs: description: "Alternative to providing organization ID. The URL of your Dagster Cloud organization." runs: using: "docker" - image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.38" + image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.39" entrypoint: "/registry_info.sh" diff --git a/actions/utils/run/action.yml b/actions/utils/run/action.yml index 39050151..500e5f65 100644 --- a/actions/utils/run/action.yml +++ b/actions/utils/run/action.yml @@ -35,7 +35,7 @@ outputs: description: "The ID of the launched run." runs: using: "docker" - image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.38" + image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.39" entrypoint: "/run.sh" args: - ${{ inputs.pr }} diff --git a/generated/gha/dagster-cloud.pex b/generated/gha/dagster-cloud.pex index 9932fa76..daece6bf 100755 Binary files a/generated/gha/dagster-cloud.pex and b/generated/gha/dagster-cloud.pex differ diff --git a/gitlab/dbt/serverless-ci-dbt.yml b/gitlab/dbt/serverless-ci-dbt.yml index 58a63a95..cedc6601 100644 --- a/gitlab/dbt/serverless-ci-dbt.yml +++ b/gitlab/dbt/serverless-ci-dbt.yml @@ -10,7 +10,7 @@ deploy-branch: stage: deploy rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: # first create the branch deployment - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix) @@ -49,7 +49,7 @@ deploy-branch: close_branch: stage: deploy - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 when: manual only: - merge_requests @@ -74,7 +74,7 @@ deploy: stage: deploy rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: # install dbt package - pip install pip --upgrade diff --git a/gitlab/hybrid-ci.yml b/gitlab/hybrid-ci.yml index 8e6bdb54..4e9e4b68 100644 --- a/gitlab/hybrid-ci.yml +++ b/gitlab/hybrid-ci.yml @@ -29,7 +29,7 @@ workflow: initialize: stage: setup - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: - export - dagster-cloud ci check --project-dir=$DAGSTER_PROJECT_DIR --dagster-cloud-yaml-path=$DAGSTER_CLOUD_YAML_PATH @@ -75,7 +75,7 @@ deploy-docker: dependencies: - build-image - initialize - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: - dagster-cloud ci set-build-output --image-tag=$IMAGE_TAG - dagster-cloud ci deploy @@ -87,7 +87,7 @@ deploy-docker-branch: dependencies: - build-image - initialize - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: - dagster-cloud ci set-build-output --image-tag=$IMAGE_TAG - dagster-cloud ci deploy @@ -97,7 +97,7 @@ deploy-docker-branch: close-branch: stage: deploy - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 when: manual only: - merge_requests diff --git a/gitlab/serverless-ci.yml b/gitlab/serverless-ci.yml index b5bf1fb7..1a74fddc 100644 --- a/gitlab/serverless-ci.yml +++ b/gitlab/serverless-ci.yml @@ -7,7 +7,7 @@ deploy-branch: stage: deploy rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: # first create the branch deployment - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix) @@ -35,7 +35,7 @@ deploy-branch: close_branch: stage: deploy - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 when: manual only: - merge_requests @@ -60,6 +60,6 @@ deploy: stage: deploy rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: - /gitlab_action/deploy.py ./dagster_cloud.yaml diff --git a/gitlab/serverless-legacy-ci.yml b/gitlab/serverless-legacy-ci.yml index 64025b82..edace635 100644 --- a/gitlab/serverless-legacy-ci.yml +++ b/gitlab/serverless-legacy-ci.yml @@ -10,7 +10,7 @@ workflow: parse-workspace: stage: setup - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: - python /gitlab_action/parse_workspace.py dagster_cloud.yaml >> build.env - cp /Dockerfile.template . @@ -23,7 +23,7 @@ parse-workspace: fetch-registry-info: stage: setup - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: dagster-cloud serverless registry-info --url $DAGSTER_CLOUD_URL/prod --api-token $DAGSTER_CLOUD_API_TOKEN | grep '=' >> registry.env artifacts: reports: @@ -53,7 +53,7 @@ deploy-docker: - build-image - parse-workspace - fetch-registry-info - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: - dagster-cloud workspace add-location --url $DAGSTER_CLOUD_URL/prod @@ -74,7 +74,7 @@ deploy-docker-branch: - build-image - parse-workspace - fetch-registry-info - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 script: - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix) - export PR_MESSAGE=$(git log -1 --format='%s') @@ -109,7 +109,7 @@ deploy-docker-branch: close_branch: stage: deploy - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.38 + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.39 when: manual only: - merge_requests diff --git a/scripts/release.py b/scripts/release.py index c84ff67c..96cc0593 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -158,6 +158,8 @@ def create_rc( check_workdir: bool = True, execute_tests: bool = True, publish_docker_action: bool = True, + dagster_internal_branch: Optional[str] = DAGSTER_INTERNAL_BRANCH_OPTION, + dagster_oss_branch: Optional[str] = DAGSTER_OSS_BRANCH_OPTION, ): if check_workdir: ensure_clean_workdir() @@ -168,7 +170,7 @@ def create_rc( error(f"Invalid version tag {version_tag}") sys.exit(1) - update_dagster_cloud_pex(None, None) + update_dagster_cloud_pex(dagster_internal_branch, dagster_oss_branch) if execute_tests: run_tests() build_docker_action(version_tag, publish_docker_action) diff --git a/src/deploy_pex.py b/src/deploy_pex.py index 207b4243..dc0b5501 100755 --- a/src/deploy_pex.py +++ b/src/deploy_pex.py @@ -100,9 +100,16 @@ def get_branch_deployment_name(project_dir): ] ) if returncode: - print("Could not determine branch deployment", flush=True) + print("Could not determine branch deployment from output:", output, flush=True) + sys.exit(1) + for line in output: + # sometimes the cmd prints warnings in addition to the branch deployment name + if re.match('[0-9a-f]+', line): + name = line.strip() + break + else: + print("Could not determine branch deployment from output: ", output, flush=True) sys.exit(1) - name = "".join(output).strip() print("Deploying to branch deployment:", name, flush=True) return name