diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e0d93a6c493..bf7b8033a54 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,13 +6,23 @@ on: - development # - '[0-9]+.[0-9]+.x' + workflow_dispatch: + inputs: + docker_registry: + description: 'Docker registry to push images to (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)' + required: true + default: 'ghcr.io/' + docker_repo: + description: 'Docker repo to push images to (default: lowercase github repository owner name)' + required: false + default: '' jobs: build-images: name: Build and push image - ${{ matrix.image-name }} runs-on: ubuntu-latest # let's not run this on every fork, change to your fork when developing - if: github.repository == 'mlrun/mlrun' + if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch' strategy: matrix: @@ -29,22 +39,44 @@ jobs: # - models-gpu-legacy steps: - uses: actions/checkout@v2 - - name: Docker login - run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u ${{ secrets.CR_USERNAME }} --password-stdin - - name: Set MLRUN_DOCKER_REPO env var - run: echo "::set-env name=MLRUN_DOCKER_REPO::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" - - name: Set GIT_HASH env var - run: echo "::set-env name=GIT_HASH::$(git rev-parse --short $GITHUB_SHA)" - name: Install curl and jq run: sudo apt-get install curl jq - - name: Set LATEST_VERSION env var - run: echo "::set-env name=LATEST_VERSION::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')" + - name: Extract git hashes and latest version + id: git_info + run: | + echo "::set-output name=mlrun_commit_hash::$(git rev-parse --short $GITHUB_SHA)" + echo "::set-output name=latest_version::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')" + - name: Set computed versions params + id: computed_params + run: | + echo "::set-output name=mlrun_version::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.mlrun_commit_hash }}" + echo "::set-output name=mlrun_docker_repo::$( \ + input_docker_repo=${{ github.event.inputs.docker_repo }} && \ + default_docker_repo=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') && \ + echo ${input_docker_repo:-`echo $default_docker_repo`})" + echo "::set-output name=mlrun_docker_registry::$( \ + input_docker_registry=${{ github.event.inputs.docker_registry }} && \ + echo ${input_docker_registry:-ghcr.io/})" + - name: Docker login + run: | + echo ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | \ + docker login ${{ steps.computed_params.outputs.mlrun_docker_registry }} -u ${{ secrets.DOCKER_REGISTRY_USERNAME }} --password-stdin - name: Pull cache, build and push image # we don't really want per-commit test image we just want to build and push the cache image so CI will be able # to use it and run much faster if: ${{ matrix.image-name != 'test' }} - run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_REPO="$MLRUN_DOCKER_REPO" MLRUN_VERSION="$LATEST_VERSION"-"$GIT_HASH" MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make push-${{ matrix.image-name }} + run: | + MLRUN_DOCKER_REGISTRY=${{ steps.computed_params.outputs.mlrun_docker_registry }} \ + MLRUN_DOCKER_REPO=${{ steps.computed_params.outputs.mlrun_docker_repo }} \ + MLRUN_VERSION=${{ steps.computed_params.outputs.mlrun_version }} \ + MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache \ + make push-${{ matrix.image-name }} - name: Pull cache, build and push test image if: ${{ matrix.image-name == 'test' }} - run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_REPO="$MLRUN_DOCKER_REPO" MLRUN_VERSION=unstable-cache MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make push-${{ matrix.image-name }} + run: | + MLRUN_DOCKER_REGISTRY=${{ steps.computed_params.outputs.mlrun_docker_registry }} \ + MLRUN_DOCKER_REPO=${{ steps.computed_params.outputs.mlrun_docker_repo }} \ + MLRUN_VERSION=unstable-cache \ + MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache \ + make push-${{ matrix.image-name }} diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index b2e7a89371a..521cc77bd64 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -11,6 +11,21 @@ on: # Run the system tests every 3 hours - cron: '0 */3 * * *' + workflow_dispatch: + inputs: + docker_registry: + description: 'Docker registry to pull images from (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)' + required: true + default: 'ghcr.io/' + docker_repo: + description: 'Docker repo to pull images from (default: mlrun)' + required: true + default: 'mlrun' + test_code_from_action: + description: 'Take tested code from action REF (default: false - take from upstream) (note that test code will be taken from the action REF anyways)' + required: true + default: 'false' + jobs: run-system-tests-ci: timeout-minutes: 60 @@ -18,7 +33,7 @@ jobs: runs-on: ubuntu-latest # let's not run this on every fork, change to your fork when developing - if: github.repository == 'mlrun/mlrun' + if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch' steps: - uses: actions/checkout@v2 @@ -30,8 +45,13 @@ jobs: run: pip install -r automation/requirements.txt && python setup.py develop - name: Install curl and jq run: sudo apt-get install curl jq - - name: Extract git hashes and latest version - id: git_info + - name: Extract git hash from action mlrun version + if: ${{ github.event.inputs.test_code_from_action == 'true' }} + id: git_action_info + run: | + echo "::set-output name=mlrun_hash::$(git rev-parse --short $GITHUB_SHA)" + - name: Extract git hashes from upstream and latest version + id: git_upstream_info run: | echo "::set-output name=mlrun_hash::$( \ cd /tmp && \ @@ -49,10 +69,19 @@ jobs: rm -rf mlrun-ui)" echo "::set-output name=latest_version::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')" - name: Set computed versions params - id: mlrun_versions + id: computed_params run: | - echo "::set-output name=mlrun::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.mlrun_hash }}" - echo "::set-output name=ui::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.ui_hash }}" + action_mlrun_hash=${{ steps.git_action_info.outputs.mlrun_hash }} && \ + upstream_mlrun_hash=${{ steps.git_upstream_info.outputs.mlrun_hash }} && \ + export mlrun_hash=${action_mlrun_hash:-`echo $upstream_mlrun_hash`} + echo "::set-output name=mlrun_version::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}-$mlrun_hash)" + echo "::set-output name=mlrun_ui_version::${{ steps.git_upstream_info.outputs.latest_version }}-${{ steps.git_upstream_info.outputs.ui_hash }}" + echo "::set-output name=mlrun_docker_repo::$( \ + input_docker_repo=${{ github.event.inputs.docker_repo }} && \ + echo ${input_docker_repo:-mlrun})" + echo "::set-output name=mlrun_docker_registry::$( \ + input_docker_registry=${{ github.event.inputs.docker_registry }} && \ + echo ${input_docker_registry:-ghcr.io/})" - name: Wait for existing runs to complete uses: softprops/turnstyle@v1 with: @@ -62,7 +91,7 @@ jobs: - name: Prepare System Test env.yaml and system run: | python automation/system_test/prepare.py run \ - ${{ steps.mlrun_versions.outputs.mlrun }} \ + ${{ steps.computed_params.outputs.mlrun_version }} \ ${{ secrets.SYSTEM_TEST_DATA_CLUSTER_IP }} \ ${{ secrets.SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }} \ ${{ secrets.SYSTEM_TEST_APP_CLUSTER_SSH_PASSWORD }} \ @@ -72,10 +101,10 @@ jobs: ${{ secrets.SYSTEM_TEST_USERNAME }} \ ${{ secrets.SYSTEM_TEST_ACCESS_KEY }} \ ${{ secrets.SYSTEM_TEST_PASSWORD }} \ - --override-image-registry "ghcr.io/" \ - --override-image-repo mlrun \ + --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}" \ + --override-image-repo ${{ steps.computed_params.outputs.mlrun_docker_repo }} \ --override-mlrun-images \ - "ghcr.io/mlrun/mlrun-api:${{ steps.mlrun_versions.outputs.mlrun }},ghcr.io/mlrun/mlrun-ui:${{ steps.mlrun_versions.outputs.ui }}" + "${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_version }},ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}" - name: Run System Tests run: make test-system diff --git a/automation/system_test/prepare.py b/automation/system_test/prepare.py index 7042542d4fb..d3b12241df8 100644 --- a/automation/system_test/prepare.py +++ b/automation/system_test/prepare.py @@ -58,16 +58,6 @@ def __init__( self._app_cluster_ssh_password = app_cluster_ssh_password self._github_access_token = github_access_token - self._override_full_image_repo = None - - if self._override_image_repo or self._override_image_registry: - - # complete with defaults if override is partial - override_registry = (self._override_image_registry or "quay.io").strip("/") - override_repo = (self._override_image_repo or "mlrun").strip("/") - - self._override_full_image_repo = f"{override_registry}/{override_repo}" - self._env_config = { "MLRUN_DBPATH": mlrun_dbpath, "V3IO_API": webapi_direct_http, @@ -326,10 +316,6 @@ def _patch_mlrun(self, provctl_path): ) mlrun_archive = f"./mlrun-{self._mlrun_version}.tar" - repo_arg = "" - if self._override_full_image_repo: - repo_arg = f"--override-image-pull-repo {self._override_full_image_repo}" - override_image_arg = "" if self._override_mlrun_images: override_image_arg = f"--override-images {self._override_mlrun_images}" @@ -340,7 +326,6 @@ def _patch_mlrun(self, provctl_path): f"--logger-file-path={str(self.Constants.workdir)}/provctl-create-patch-{time_string}.log", "create-patch", "appservice", - repo_arg, override_image_arg, "mlrun", self._mlrun_version,