Adds possibility of forcing upgrade constraint by setting a label (#1…

…2635) You can now set a label on PR that will force upgrading to latest dependencies in your PR. If committer sets an "upgrade to latest dependencies" label, it will cause the PR to upgrade all dependencies to latest versions of dependencies matching setup.py + setup.cfg configuration. (cherry picked from commit 8b9d52f)
apache · Dec 3, 2020 · a4a825b · a4a825b
1 parent 416b125
commit a4a825b
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 37 deletions.
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
@@ -30,7 +30,6 @@ env:
   SKIP_CHECK_REMOTE_IMAGE: "true"
   DB_RESET: "true"
   VERBOSE: "true"
-  UPGRADE_TO_LATEST_CONSTRAINTS: false
   USE_GITHUB_REGISTRY: "true"
   GITHUB_REPOSITORY: ${{ github.repository }}
   GITHUB_USERNAME: ${{ github.actor }}
@@ -57,7 +56,6 @@ jobs:
       sourceEvent: ${{ steps.source-run-info.outputs.sourceEvent }}
       cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
       buildImages: ${{ steps.build-images.outputs.buildImages }}
-      upgradeToLatestConstraints: ${{ steps.upgrade-constraints.outputs.upgradeToLatestConstraints }}
     steps:
       - name: "Get information about the original trigger of the run"
         uses: potiuk/get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec  # v1_3
@@ -153,15 +151,6 @@ jobs:
           else
               echo "::set-output name=docker-cache::pulled"
           fi
-      - name: "Set upgrade to latest constraints"
-        id: upgrade-constraints
-        run: |
-          if [[ ${{ steps.cancel.outputs.sourceEvent == 'push' ||
-              steps.cancel.outputs.sourceEvent == 'scheduled' }} == 'true' ]]; then
-              echo "::set-output name=upgradeToLatestConstraints::${{ github.sha }}"
-          else
-              echo "::set-output name=upgradeToLatestConstraints::false"
-          fi
       - name: "Cancel all duplicated 'Build Image' runs"
         # We find duplicates of all "Build Image" runs - due to a missing feature
         # in GitHub Actions, we have to use Job names to match Event/Repo/Branch matching
@@ -198,6 +187,7 @@ jobs:
       GITHUB_CONTEXT: ${{ toJson(github) }}
     outputs:
       pythonVersions: ${{ steps.selective-checks.python-versions }}
+      upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }}
       allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
       defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
       run-tests: ${{ steps.selective-checks.outputs.run-tests }}
@@ -243,12 +233,12 @@ jobs:
         id: selective-checks
         env:
           EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
-          INCOMING_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
+          TARGET_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
           PR_LABELS: ${{ needs.cancel-workflow-runs.outputs.pullRequestLabels }}
         run: |
           if [[ ${EVENT_NAME} == "pull_request" ]]; then
             # Run selective checks
-            ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+            ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
           else
             # Run all checks
             ./scripts/ci/selective_ci_checks.sh
@@ -273,7 +263,7 @@ jobs:
       BACKEND: postgres
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }}
-      UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.cancel-workflow-runs.outputs.upgradeToLatestConstraints }}
+      UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
       DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
     steps:
       - name: >

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -35,7 +35,6 @@ env:
   SKIP_CHECK_REMOTE_IMAGE: "true"
   DB_RESET: "true"
   VERBOSE: "true"
-  UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }}
   DOCKER_CACHE: "pulled"
   USE_GITHUB_REGISTRY: "true"
   GITHUB_REPOSITORY: ${{ github.repository }}
@@ -69,6 +68,7 @@ jobs:
       GITHUB_CONTEXT: ${{ toJson(github) }}
     outputs:
       waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
+      upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }}
       pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
       pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
       defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
@@ -131,12 +131,12 @@ jobs:
         id: selective-checks
         env:
           EVENT_NAME: ${{ github.event_name }}
-          INCOMING_COMMIT_SHA: ${{ github.sha }}
+          TARGET_COMMIT_SHA: ${{ github.sha }}
           PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
         run: |
           if [[ ${EVENT_NAME} == "pull_request" ]]; then
             # Run selective checks
-            ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+            ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
           else
             # Run all checks
             ./scripts/ci/selective_ci_checks.sh
@@ -150,6 +150,7 @@ jobs:
     if: needs.build-info.outputs.image-build == 'true'
     env:
       BACKEND: sqlite
+      UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -568,7 +569,8 @@ jobs:
     needs: [build-info]
     env:
       BACKEND: sqlite
-      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+      PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+      UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }}
     if: needs.build-info.outputs.image-build == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -40,11 +40,11 @@ jobs:
         id: selective-checks
         env:
           EVENT_NAME: ${{ github.event_name }}
-          INCOMING_COMMIT_SHA: ${{ github.sha }}
+          TARGET_COMMIT_SHA: ${{ github.sha }}
         run: |
           if [[ ${EVENT_NAME} == "pull_request" ]]; then
             # Run selective checks
-            ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+            ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
           else
             # Run all checks
             ./scripts/ci/selective_ci_checks.sh

diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml
@@ -75,12 +75,12 @@ jobs:
         id: selective-checks
         env:
           EVENT_NAME: ${{ steps.source-run-info.outputs.sourceEvent }}
-          INCOMING_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }}
+          TARGET_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }}
           PR_LABELS: ${{ steps.source-run-info.outputs.pullRequestLabels }}
         run: |
           if [[ ${EVENT_NAME} == "pull_request_review" ]]; then
             # Run selective checks
-            ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+            ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}"
           else
             # Run all checks
             ./scripts/ci/selective_ci_checks.sh

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -321,6 +321,11 @@ Step 4: Prepare PR
        the "full tests needed" label is set for your PR. Additional check is set that prevents from
        accidental merging of the request until full matrix of tests succeeds for the PR.
 
+     * when your change has "upgrade to latest dependencies" label set, constraints will be automatically
+       upgraded to latest constraints matching your setup.py. This is useful in case you want to force
+       upgrade to a latest version of dependencies. You can ask committers to set the label for you
+       when you need it in your PR.
+
    More details about the PR workflow be found in `PULL_REQUEST_WORKFLOW.rst <PULL_REQUEST_WORKFLOW.rst>`_.
 
 

diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh
@@ -34,16 +34,39 @@ if [[ ${PR_LABELS=} == *"full tests needed"* ]]; then
     echo
     echo "Found the right PR labels in '${PR_LABELS=}': 'full tests needed''"
     echo
-    FULL_TESTS_NEEDED="true"
+    FULL_TESTS_NEEDED_LABEL="true"
 else
     echo
     echo "Did not find the right PR labels in '${PR_LABELS=}': 'full tests needed'"
     echo
-    FULL_TESTS_NEEDED="false"
+    FULL_TESTS_NEEDED_LABEL="false"
+fi
+
+if [[ ${PR_LABELS=} == *"upgrade to latest dependencies"* ]]; then
+    echo
+    echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies''"
+    echo
+    UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="true"
+else
+    echo
+    echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies'"
+    echo
+    UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="false"
 fi
 
 function output_all_basic_variables() {
-    if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+    if [[ "${UPGRADE_TO_LATEST_CONSTRAINTS_LABEL}" == "true" ||
+            ${EVENT_NAME} == 'push' || ${EVENT_NAME} == "scheduled" ]]; then
+        # Trigger upgrading to latest constraints where label is set or when
+        # SHA of the merge commit triggers rebuilding layer in the docker image
+        # Each build that upgrades to latest constraints will get truly latest constraints, not those
+        # Cached in the image this way
+        initialization::ga_output upgrade-to-latest-constraints "${INCOMING_COMMIT_SHA}"
+    else
+        initialization::ga_output upgrade-to-latest-constraints "false"
+    fi
+
+    if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
         initialization::ga_output python-versions \
             "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")"
         initialization::ga_output all-python-versions \
@@ -60,7 +83,7 @@ function output_all_basic_variables() {
     fi
     initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
 
-    if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+    if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
         initialization::ga_output kubernetes-versions \
             "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_VERSIONS[@]}")"
     else
@@ -73,7 +96,7 @@ function output_all_basic_variables() {
         "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_MODES[@]}")"
     initialization::ga_output default-kubernetes-mode "${KUBERNETES_MODE}"
 
-    if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+    if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
         initialization::ga_output postgres-versions \
             "$(initialization::parameters_to_json "${CURRENT_POSTGRES_VERSIONS[@]}")"
     else
@@ -82,7 +105,7 @@ function output_all_basic_variables() {
     fi
     initialization::ga_output default-postgres-version "${POSTGRES_VERSION}"
 
-    if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+    if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
         initialization::ga_output mysql-versions \
             "$(initialization::parameters_to_json "${CURRENT_MYSQL_VERSIONS[@]}")"
     else
@@ -100,7 +123,7 @@ function output_all_basic_variables() {
         "$(initialization::parameters_to_json "${CURRENT_HELM_VERSIONS[@]}")"
     initialization::ga_output default-helm-version "${HELM_VERSION}"
 
-    if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then
+    if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then
         initialization::ga_output postgres-exclude '[{ "python-version": "3.6" }]'
         initialization::ga_output mysql-exclude '[{ "python-version": "3.7" }]'
         initialization::ga_output sqlite-exclude '[{ "python-version": "3.8" }]'
@@ -114,9 +137,6 @@ function output_all_basic_variables() {
 }
 
 function get_changed_files() {
-    INCOMING_COMMIT_SHA="${1}"
-    readonly INCOMING_COMMIT_SHA
-
     echo
     echo "Incoming commit SHA: ${INCOMING_COMMIT_SHA}"
     echo
@@ -414,22 +434,28 @@ if (($# < 1)); then
     echo
     echo "No Commit SHA - running all tests (likely direct master merge, or scheduled run)!"
     echo
-    # override FULL_TESTS_NEEDED in master/scheduled run
-    FULL_TESTS_NEEDED="true"
-    readonly FULL_TESTS_NEEDED
+    INCOMING_COMMIT_SHA=""
+    readonly INCOMING_COMMIT_SHA
+    # override FULL_TESTS_NEEDED_LABEL in master/scheduled run
+    FULL_TESTS_NEEDED_LABEL="true"
+    readonly FULL_TESTS_NEEDED_LABEL
     output_all_basic_variables
     set_outputs_run_everything_and_exit
+else
+    INCOMING_COMMIT_SHA="${1}"
+    readonly INCOMING_COMMIT_SHA
 fi
 
-readonly FULL_TESTS_NEEDED
+
+readonly FULL_TESTS_NEEDED_LABEL
 output_all_basic_variables
 
 image_build_needed="false"
 docs_build_needed="false"
 tests_needed="false"
 kubernetes_tests_needed="false"
 
-get_changed_files "${1}"
+get_changed_files
 run_all_tests_if_environment_files_changed
 check_if_docs_should_be_generated
 check_if_helm_tests_should_be_run