Enable to trigger workflows manually (#468)

mlrun · Oct 8, 2020 · 65a00a9 · 65a00a9
1 parent a6e91db
commit 65a00a9
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 36 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -6,13 +6,23 @@ on:
     - development
 #    - '[0-9]+.[0-9]+.x'
 
+  workflow_dispatch:
+    inputs:
+      docker_registry:
+        description: 'Docker registry to push images to (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)'
+        required: true
+        default: 'ghcr.io/'
+      docker_repo:
+        description: 'Docker repo to push images to (default: lowercase github repository owner name)'
+        required: false
+        default: ''
 jobs:
   build-images:
     name: Build and push image - ${{ matrix.image-name }}
     runs-on: ubuntu-latest
 
     # let's not run this on every fork, change to your fork when developing
-    if: github.repository == 'mlrun/mlrun'
+    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'
 
     strategy:
       matrix:
@@ -29,22 +39,44 @@ jobs:
 #        - models-gpu-legacy
     steps:
     - uses: actions/checkout@v2
-    - name: Docker login
-      run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u ${{ secrets.CR_USERNAME }} --password-stdin
-    - name: Set MLRUN_DOCKER_REPO env var
-      run: echo "::set-env name=MLRUN_DOCKER_REPO::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')"
-    - name: Set GIT_HASH env var
-      run: echo "::set-env name=GIT_HASH::$(git rev-parse --short $GITHUB_SHA)"
     - name: Install curl and jq
       run: sudo apt-get install curl jq
-    - name: Set LATEST_VERSION env var
-      run: echo "::set-env name=LATEST_VERSION::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')"
+    - name: Extract git hashes and latest version
+      id: git_info
+      run: |
+        echo "::set-output name=mlrun_commit_hash::$(git rev-parse --short $GITHUB_SHA)"
+        echo "::set-output name=latest_version::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')"
+    - name: Set computed versions params
+      id: computed_params
+      run: |
+        echo "::set-output name=mlrun_version::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.mlrun_commit_hash }}"
+        echo "::set-output name=mlrun_docker_repo::$( \
+          input_docker_repo=${{ github.event.inputs.docker_repo }} && \
+          default_docker_repo=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') && \
+          echo ${input_docker_repo:-`echo $default_docker_repo`})"
+        echo "::set-output name=mlrun_docker_registry::$( \
+          input_docker_registry=${{ github.event.inputs.docker_registry }} && \
+          echo ${input_docker_registry:-ghcr.io/})"
+    - name: Docker login
+      run: |
+        echo ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | \
+        docker login ${{ steps.computed_params.outputs.mlrun_docker_registry }} -u ${{ secrets.DOCKER_REGISTRY_USERNAME }} --password-stdin
     - name: Pull cache, build and push image
 
       # we don't really want per-commit test image we just want to build and push the cache image so CI will be able
       # to use it and run much faster
       if: ${{ matrix.image-name != 'test' }}
-      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_REPO="$MLRUN_DOCKER_REPO" MLRUN_VERSION="$LATEST_VERSION"-"$GIT_HASH" MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make push-${{ matrix.image-name }}
+      run: |
+        MLRUN_DOCKER_REGISTRY=${{ steps.computed_params.outputs.mlrun_docker_registry }} \
+        MLRUN_DOCKER_REPO=${{ steps.computed_params.outputs.mlrun_docker_repo }} \
+        MLRUN_VERSION=${{ steps.computed_params.outputs.mlrun_version }} \
+        MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache \
+        make push-${{ matrix.image-name }}
     - name: Pull cache, build and push test image
       if: ${{ matrix.image-name == 'test' }}
-      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_REPO="$MLRUN_DOCKER_REPO" MLRUN_VERSION=unstable-cache MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make push-${{ matrix.image-name }}
+      run: |
+        MLRUN_DOCKER_REGISTRY=${{ steps.computed_params.outputs.mlrun_docker_registry }} \
+        MLRUN_DOCKER_REPO=${{ steps.computed_params.outputs.mlrun_docker_repo }} \
+        MLRUN_VERSION=unstable-cache \
+        MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache \
+        make push-${{ matrix.image-name }}
diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -11,14 +11,29 @@ on:
     # Run the system tests every 3 hours
     - cron:  '0 */3 * * *'
 
+  workflow_dispatch:
+    inputs:
+      docker_registry:
+        description: 'Docker registry to pull images from (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)'
+        required: true
+        default: 'ghcr.io/'
+      docker_repo:
+        description: 'Docker repo to pull images from (default: mlrun)'
+        required: true
+        default: 'mlrun'
+      test_code_from_action:
+        description: 'Take tested code from action REF (default: false - take from upstream) (note that test code will be taken from the action REF anyways)'
+        required: true
+        default: 'false'
+
 jobs:
   run-system-tests-ci:
     timeout-minutes: 60
     name: Run System Tests
     runs-on: ubuntu-latest
 
     # let's not run this on every fork, change to your fork when developing
-    if: github.repository == 'mlrun/mlrun'
+    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'
 
     steps:
     - uses: actions/checkout@v2
@@ -30,8 +45,13 @@ jobs:
       run: pip install -r automation/requirements.txt && python setup.py develop
     - name: Install curl and jq
       run: sudo apt-get install curl jq
-    - name: Extract git hashes and latest version
-      id: git_info
+    - name: Extract git hash from action mlrun version
+      if: ${{ github.event.inputs.test_code_from_action == 'true' }}
+      id: git_action_info
+      run: |
+        echo "::set-output name=mlrun_hash::$(git rev-parse --short $GITHUB_SHA)"
+    - name: Extract git hashes from upstream and latest version
+      id: git_upstream_info
       run: |
         echo "::set-output name=mlrun_hash::$( \
           cd /tmp && \
@@ -49,10 +69,19 @@ jobs:
           rm -rf mlrun-ui)"
         echo "::set-output name=latest_version::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')"
     - name: Set computed versions params
-      id: mlrun_versions
+      id: computed_params
       run: |
-        echo "::set-output name=mlrun::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.mlrun_hash }}"
-        echo "::set-output name=ui::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.ui_hash }}"
+        action_mlrun_hash=${{ steps.git_action_info.outputs.mlrun_hash }} && \
+        upstream_mlrun_hash=${{ steps.git_upstream_info.outputs.mlrun_hash }} && \
+        export mlrun_hash=${action_mlrun_hash:-`echo $upstream_mlrun_hash`}
+        echo "::set-output name=mlrun_version::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}-$mlrun_hash)"
+        echo "::set-output name=mlrun_ui_version::${{ steps.git_upstream_info.outputs.latest_version }}-${{ steps.git_upstream_info.outputs.ui_hash }}"
+        echo "::set-output name=mlrun_docker_repo::$( \
+          input_docker_repo=${{ github.event.inputs.docker_repo }} && \
+          echo ${input_docker_repo:-mlrun})"
+        echo "::set-output name=mlrun_docker_registry::$( \
+          input_docker_registry=${{ github.event.inputs.docker_registry }} && \
+          echo ${input_docker_registry:-ghcr.io/})"
     - name: Wait for existing runs to complete
       uses: softprops/turnstyle@v1
       with:
@@ -62,7 +91,7 @@ jobs:
     - name: Prepare System Test env.yaml and system
       run: |
         python automation/system_test/prepare.py run \
-          ${{ steps.mlrun_versions.outputs.mlrun }} \
+          ${{ steps.computed_params.outputs.mlrun_version }} \
           ${{ secrets.SYSTEM_TEST_DATA_CLUSTER_IP }} \
           ${{ secrets.SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }} \
           ${{ secrets.SYSTEM_TEST_APP_CLUSTER_SSH_PASSWORD }} \
@@ -72,10 +101,10 @@ jobs:
           ${{ secrets.SYSTEM_TEST_USERNAME }} \
           ${{ secrets.SYSTEM_TEST_ACCESS_KEY }} \
           ${{ secrets.SYSTEM_TEST_PASSWORD }} \
-          --override-image-registry "ghcr.io/" \
-          --override-image-repo mlrun \
+          --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
+          --override-image-repo ${{ steps.computed_params.outputs.mlrun_docker_repo }} \
           --override-mlrun-images \
-          "ghcr.io/mlrun/mlrun-api:${{ steps.mlrun_versions.outputs.mlrun }},ghcr.io/mlrun/mlrun-ui:${{ steps.mlrun_versions.outputs.ui }}"
+          "${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_version }},ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}"
 
     - name: Run System Tests
       run: make test-system
diff --git a/automation/system_test/prepare.py b/automation/system_test/prepare.py
@@ -58,16 +58,6 @@ def __init__(
         self._app_cluster_ssh_password = app_cluster_ssh_password
         self._github_access_token = github_access_token
 
-        self._override_full_image_repo = None
-
-        if self._override_image_repo or self._override_image_registry:
-
-            # complete with defaults if override is partial
-            override_registry = (self._override_image_registry or "quay.io").strip("/")
-            override_repo = (self._override_image_repo or "mlrun").strip("/")
-
-            self._override_full_image_repo = f"{override_registry}/{override_repo}"
-
         self._env_config = {
             "MLRUN_DBPATH": mlrun_dbpath,
             "V3IO_API": webapi_direct_http,
@@ -326,10 +316,6 @@ def _patch_mlrun(self, provctl_path):
         )
         mlrun_archive = f"./mlrun-{self._mlrun_version}.tar"
 
-        repo_arg = ""
-        if self._override_full_image_repo:
-            repo_arg = f"--override-image-pull-repo {self._override_full_image_repo}"
-
         override_image_arg = ""
         if self._override_mlrun_images:
             override_image_arg = f"--override-images {self._override_mlrun_images}"
@@ -340,7 +326,6 @@ def _patch_mlrun(self, provctl_path):
                 f"--logger-file-path={str(self.Constants.workdir)}/provctl-create-patch-{time_string}.log",
                 "create-patch",
                 "appservice",
-                repo_arg,
                 override_image_arg,
                 "mlrun",
                 self._mlrun_version,