Add caching to CI (#449)

mlrun · Sep 25, 2020 · 0a71cb0 · 0a71cb0
1 parent b7d7aac
commit 0a71cb0
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 22 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -23,18 +23,28 @@ jobs:
         - base-legacy
         - models
         - models-legacy
+        - test
 # The GPU images are too big (~24GB) GH action runner has only 14GB
 #        - models-gpu
 #        - models-gpu-legacy
     steps:
     - uses: actions/checkout@v2
     - name: Docker login
       run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u ${{ secrets.CR_USERNAME }} --password-stdin
+    - name: Set MLRUN_DOCKER_REPO env var
+      run: echo "::set-env name=MLRUN_DOCKER_REPO::$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')"
     - name: Set GIT_HASH env var
       run: echo "::set-env name=GIT_HASH::$(git rev-parse --short $GITHUB_SHA)"
     - name: Install curl and jq
       run: sudo apt-get install curl jq
     - name: Set LATEST_VERSION env var
       run: echo "::set-env name=LATEST_VERSION::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')"
     - name: Pull cache, build and push image
-      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_VERSION="$LATEST_VERSION"-"$GIT_HASH" MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make push-${{ matrix.image-name }}
+
+      # we don't really want per-commit test image we just want to build and push the cache image so CI will be able
+      # to use it and run much faster
+      if: ${{ matrix.image-name != 'test' }}
+      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_REPO="$MLRUN_DOCKER_REPO" MLRUN_VERSION="$LATEST_VERSION"-"$GIT_HASH" MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make push-${{ matrix.image-name }}
+    - name: Pull cache, build and push test image
+      if: ${{ matrix.image-name == 'test' }}
+      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_REPO="$MLRUN_DOCKER_REPO" MLRUN_VERSION=unstable-cache MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make push-${{ matrix.image-name }}
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -24,38 +24,35 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
-#    - uses: actions/cache@v1
-#      id: cache-dev-deps
-#      with:
-#        path: .venv
-#        key: venv-dev-${{ runner.os }}-py-${{ matrix.python-version }}-poetry-${{ hashFiles('poetry.lock') }}
-#        restore-keys: |
-#          venv-dev-${{ runner.os }}-py-${{ matrix.python-version }}-poetry-
-#          venv-dev-${{ runner.os }}-py-${{ matrix.python-version }}-
-#          venv-dev-${{ runner.os }}-py-
-#          venv-dev-${{ runner.os }}-
+    - uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/dev-requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python-version }}-
+          ${{ runner.os }}-pip-
     - name: Install dependencies
-#      if: steps.cache-dev-deps.outputs.cache-hit != 'true'
       run: |
         python -m pip install --upgrade pip
         pip install -r dev-requirements.txt
     - name: Lint
       run: make lint
+
   tests:
     name: Run Dockerized Tests
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: Run Dockerized tests
-      run: make test-dockerized
+      run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make test-dockerized
 
   docs:
     name: Build Project Documentation
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Generate HTML docs
-        run: make html-docs-dockerized
+        run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=unstable-cache make html-docs-dockerized
       - name: Upload generated docs
         uses: actions/upload-artifact@v2
         with:

diff --git a/Makefile b/Makefile
@@ -323,14 +323,26 @@ push-api: api ## Push api docker image
 	$(MLRUN_API_CACHE_IMAGE_PUSH_COMMAND)
 
 
-MLRUN_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test:$(MLRUN_DOCKER_TAG)
+MLRUN_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test
+MLRUN_TEST_IMAGE_NAME_TAGGED := $(MLRUN_TEST_IMAGE_NAME):$(MLRUN_DOCKER_TAG)
+MLRUN_TEST_CACHE_IMAGE_NAME_TAGGED := $(MLRUN_TEST_IMAGE_NAME):$(MLRUN_DOCKER_CACHE_FROM_TAG)
+MLRUN_TEST_IMAGE_DOCKER_CACHE_FROM_FLAG := $(if $(MLRUN_DOCKER_CACHE_FROM_TAG),--cache-from $(strip $(MLRUN_TEST_CACHE_IMAGE_NAME_TAGGED)),)
+MLRUN_TEST_CACHE_IMAGE_PULL_COMMAND := $(if $(MLRUN_DOCKER_CACHE_FROM_TAG),docker pull $(MLRUN_TEST_CACHE_IMAGE_NAME_TAGGED) || true,)
+MLRUN_TEST_CACHE_IMAGE_PUSH_COMMAND := $(if $(MLRUN_DOCKER_CACHE_FROM_TAG),docker tag $(MLRUN_TEST_IMAGE_NAME_TAGGED) $(MLRUN_TEST_CACHE_IMAGE_NAME_TAGGED) && docker push $(MLRUN_TEST_CACHE_IMAGE_NAME_TAGGED),)
 
 .PHONY: build-test
 build-test: update-version-file ## Build test docker image
+	$(MLRUN_TEST_CACHE_IMAGE_PULL_COMMAND)
 	docker build \
 		--file dockerfiles/test/Dockerfile \
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
-		--tag $(MLRUN_TEST_IMAGE_NAME) .
+		$(MLRUN_TEST_IMAGE_DOCKER_CACHE_FROM_FLAG) \
+		--tag $(MLRUN_TEST_IMAGE_NAME_TAGGED) .
+
+.PHONY: push-test
+push-test: build-test ## Push test docker image
+	docker push $(MLRUN_TEST_IMAGE_NAME_TAGGED)
+	$(MLRUN_TEST_CACHE_IMAGE_PUSH_COMMAND)
 
 MLRUN_SYSTEM_TEST_IMAGE_NAME := $(MLRUN_DOCKER_IMAGE_PREFIX)/test-system:$(MLRUN_DOCKER_TAG)
 
@@ -341,10 +353,6 @@ build-test-system: update-version-file ## Build system tests docker image
 		--build-arg MLRUN_PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) \
 		--tag $(MLRUN_SYSTEM_TEST_IMAGE_NAME) .
 
-.PHONY: push-test
-push-test: build-test ## Push test docker image
-	docker push $(MLRUN_TEST_IMAGE_NAME)
-
 .PHONY: package-wheel
 package-wheel: clean update-version-file ## Build python package wheel
 	python setup.py bdist_wheel
@@ -372,7 +380,7 @@ test-dockerized: build-test ## Run mlrun tests in docker container
 		--network='host' \
 		-v /tmp:/tmp \
 		-v /var/run/docker.sock:/var/run/docker.sock \
-		$(MLRUN_TEST_IMAGE_NAME) make test
+		$(MLRUN_TEST_IMAGE_NAME_TAGGED) make test
 
 .PHONY: test
 test: clean ## Run mlrun tests
@@ -399,7 +407,7 @@ html-docs-dockerized: build-test ## Build html docs dockerized
 	docker run \
 		--rm \
 		-v $(PWD)/docs/_build:/mlrun/docs/_build \
-		$(MLRUN_TEST_IMAGE_NAME) \
+		$(MLRUN_TEST_IMAGE_NAME_TAGGED) \
 		make html-docs
 
 .PHONY: fmt