[Requirements] Bump pip to 21.2.x and python to 3.7.11 (#1418)

.github/workflows/build.yaml

@@ -57,7 +57,7 @@ jobs:
     - name: Set computed versions params
       id: computed_params
       run: |
-        echo "::set-output name=mlrun_version::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.mlrun_commit_hash }}"
+        echo "::set-output name=mlrun_version::${{ steps.git_info.outputs.latest_version }}+${{ steps.git_info.outputs.mlrun_commit_hash }}"
         echo "::set-output name=mlrun_docker_repo::$( \
           input_docker_repo=${{ github.event.inputs.docker_repo }} && \
           default_docker_repo=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') && \

.github/workflows/ci.yaml

@@ -33,7 +33,7 @@ jobs:
           ${{ runner.os }}-pip-
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip~=20.2.0
+        python -m pip install --upgrade pip~=21.2.0
         pip install -r dev-requirements.txt
     - name: Lint
       run: make lint

.github/workflows/system-tests-enterprise.yml

@@ -105,7 +105,8 @@ jobs:
         upstream_mlrun_ui_hash=${{ steps.git_upstream_info.outputs.ui_hash }} && \
         export ui_hash=${action_mlrun_ui_hash:-`echo $upstream_mlrun_ui_hash`}
         echo "::set-output name=ui_hash::$(echo $ui_hash)"
-        echo "::set-output name=mlrun_version::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}-$mlrun_hash)"
+        echo "::set-output name=mlrun_version::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}+$mlrun_hash)"
+        echo "::set-output name=mlrun_docker_tag::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}-$mlrun_hash)"
         echo "::set-output name=mlrun_ui_version::${{ steps.git_upstream_info.outputs.latest_version }}-$ui_hash"
         echo "::set-output name=mlrun_docker_repo::$( \
           input_docker_repo=${{ github.event.inputs.docker_repo }} && \
@@ -146,7 +147,7 @@ jobs:
           --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
           --override-image-repo ${{ steps.computed_params.outputs.mlrun_docker_repo }} \
           --override-mlrun-images \
-          "${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_version }},ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}"
+          "${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_docker_tag }},ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}"
 
     - name: Prepare System Test env.yaml and 0.5.x system
       if: ${{ steps.git_info.outputs.branch == '0.5.x' }}
@@ -167,7 +168,7 @@ jobs:
           --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
           --override-image-repo ${{ steps.computed_params.outputs.mlrun_docker_repo }} \
           --override-mlrun-images \
-          "${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_version }},ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}"
+          "${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_docker_tag }},ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}"
 
     - name: Run System Tests
       run: |

.github/workflows/system-tests-opensource.yml

@@ -114,7 +114,8 @@ jobs:
         upstream_mlrun_ui_hash=${{ steps.git_upstream_info.outputs.ui_hash }} && \
         export ui_hash=${action_mlrun_ui_hash:-`echo $upstream_mlrun_ui_hash`}
         echo "::set-output name=ui_hash::$(echo $ui_hash)"
-        echo "::set-output name=mlrun_version::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}-$mlrun_hash)"
+        echo "::set-output name=mlrun_version::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}+$mlrun_hash)"
+        echo "::set-output name=mlrun_docker_tag::$(echo ${{ steps.git_upstream_info.outputs.latest_version }}-$mlrun_hash)"
         echo "::set-output name=mlrun_ui_version::${{ steps.git_upstream_info.outputs.latest_version }}-$ui_hash"
         echo "::set-output name=mlrun_docker_repo::$( \
           input_docker_repo=${{ github.event.inputs.docker_repo }} && \
@@ -156,7 +157,7 @@ jobs:
             --set global.registry.secretName="" \
             --set global.externalHostAddress=$(minikube ip) \
             --set mlrun.api.image.repository=${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api \
-            --set mlrun.api.image.tag=${{ steps.computed_params.outputs.mlrun_version }} \
+            --set mlrun.api.image.tag=${{ steps.computed_params.outputs.mlrun_docker_tag }} \
             --set mlrun.ui.image.repository=ghcr.io/mlrun/mlrun-ui \
             --set mlrun.ui.image.tag=${{ steps.computed_params.outputs.mlrun_ui_version }} \
             --set mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__BUILDER__MLRUN_VERSION_SPECIFIER="mlrun[complete] @ git+https://github.com/mlrun/mlrun@${{ steps.computed_params.outputs.mlrun_hash }}" \

Makefile

@@ -13,7 +13,16 @@
 # limitations under the License.
 
 MLRUN_VERSION ?= unstable
-MLRUN_DOCKER_TAG ?= $(MLRUN_VERSION)
+# pip requires the python version to be according to some regex (so "unstable" is not valid for example) this regex only
+# allows us to have free text (like unstable) after the "+". on the contrary in a docker tag "+" is not a valid
+# character so we're doing best effort - if the provided version doesn't look valid (like unstable), we prefix the
+# version for the python package with 0.0.0+
+# if the provided version includes a "+" we replace it with "-" for the docker tag
+MLRUN_DOCKER_TAG ?= $(shell echo "$(MLRUN_VERSION)" | sed 's/\+/\-/g')
+MLRUN_PYTHON_PACKAGE_VERSION ?= $(MLRUN_VERSION)
+ifeq ($(shell echo "$(MLRUN_VERSION)" | grep -E "^[0-9]+\.[0-9]+\.[0-9]+.*$$"),) # empty result from egrep
+	MLRUN_PYTHON_PACKAGE_VERSION := 0.0.0+$(MLRUN_VERSION)
+endif
 MLRUN_DOCKER_REPO ?= mlrun
 # empty by default (dockerhub), can be set to something like "quay.io/".
 # This will be used to tag the images built using this makefile
@@ -22,7 +31,7 @@ MLRUN_DOCKER_REGISTRY ?=
 # disable caching)
 MLRUN_NO_CACHE ?=
 MLRUN_ML_DOCKER_IMAGE_NAME_PREFIX ?= ml-
-MLRUN_PYTHON_VERSION ?= 3.7.9
+MLRUN_PYTHON_VERSION ?= 3.7.11
 MLRUN_LEGACY_ML_PYTHON_VERSION ?= 3.6.12
 MLRUN_CACHE_DATE ?= $(shell date +%s)
 # empty by default, can be set to something like "tag-name" which will cause to:
@@ -59,7 +68,7 @@ all:
 
 .PHONY: install-requirements
 install-requirements: ## Install all requirements needed for development
-	python -m pip install --upgrade $(MLRUN_PIP_NO_CACHE_FLAG) pip~=20.2.0
+	python -m pip install --upgrade $(MLRUN_PIP_NO_CACHE_FLAG) pip~=21.2.0
 	python -m pip install \
 		$(MLRUN_PIP_NO_CACHE_FLAG) \
 		-r requirements.txt \
@@ -114,7 +123,7 @@ endif
 
 .PHONY: update-version-file
 update-version-file: ## Update the version file
-	python ./automation/version/version_file.py --mlrun-version $(MLRUN_VERSION)
+	python ./automation/version/version_file.py --mlrun-version $(MLRUN_PYTHON_PACKAGE_VERSION)
 
 .PHONY: build
 build: docker-images package-wheel ## Build all artifacts

automation/package_test/test.py

@@ -137,7 +137,7 @@ def _install_extra(self, extra):
             "Installing extra", extra=extra,
         )
         self._run_command(
-            "python -m pip install --upgrade pip~=20.2.0", run_in_venv=True,
+            "python -m pip install --upgrade pip~=21.2.0", run_in_venv=True,
         )
 
         self._run_command(

automation/version/version_file.py

@@ -16,7 +16,9 @@
 def main():
     parser = argparse.ArgumentParser(description="Create or update the version file")
 
-    parser.add_argument("--mlrun-version", type=str, required=False, default="unstable")
+    parser.add_argument(
+        "--mlrun-version", type=str, required=False, default="0.0.0+unstable"
+    )
 
     args = parser.parse_args()
 

dockerfiles/base/Dockerfile

@@ -1,4 +1,4 @@
-ARG MLRUN_PYTHON_VERSION=3.7
+ARG MLRUN_PYTHON_VERSION=3.7.11
 
 FROM quay.io/mlrun/python:${MLRUN_PYTHON_VERSION}-slim
 
@@ -30,14 +30,18 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86
 
 ENV PATH /opt/conda/bin:$PATH
 
-RUN conda install python=${MLRUN_PYTHON_VERSION} pip~=20.2.0 && conda clean -aqy
+RUN conda install python=${MLRUN_PYTHON_VERSION} pip~=21.2.0 && conda clean -aqy
 
-COPY ./dockerfiles/base/requirements.txt ./base-image-requirements.txt
 COPY ./dockerfiles/mlrun/requirements.txt ./mlrun-image-requirements.txt
 COPY ./extras-requirements.txt ./extras-requirements.txt
 COPY ./requirements.txt ./
 RUN python -m pip install \
     -r requirements.txt \
     -r extras-requirements.txt \
-    -r mlrun-image-requirements.txt \
+    -r mlrun-image-requirements.txt
+
+# if we give all the requirements together pip resolver go nuts so separating between the package and the image
+# requirements
+COPY ./dockerfiles/base/requirements.txt ./base-image-requirements.txt
+RUN python -m pip install \
     -r base-image-requirements.txt

dockerfiles/jupyter/Dockerfile

@@ -8,7 +8,7 @@ USER $NB_UID
 
 ENV PIP_NO_CACHE_DIR=1
 
-RUN python -m pip install --upgrade pip~=20.2.0
+RUN python -m pip install --upgrade pip~=21.2.0
 
 WORKDIR $HOME
 

dockerfiles/mlrun-api/Dockerfile

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG MLRUN_PYTHON_VERSION=3.7
+ARG MLRUN_PYTHON_VERSION=3.7.11
 
 FROM quay.io/mlrun/python:${MLRUN_PYTHON_VERSION}-slim
 
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   vim \
  && rm -rf /var/lib/apt/lists/*
 
-RUN python -m pip install --upgrade pip~=20.2.0
+RUN python -m pip install --upgrade pip~=21.2.0
 
 WORKDIR /mlrun
 

dockerfiles/mlrun/Dockerfile

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG MLRUN_PYTHON_VERSION=3.7
+ARG MLRUN_PYTHON_VERSION=3.7.11
 
 FROM quay.io/mlrun/python:${MLRUN_PYTHON_VERSION}-slim
 
@@ -25,7 +25,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
 
 WORKDIR /mlrun
 
-RUN python -m pip install --upgrade pip~=20.2.0
+RUN python -m pip install --upgrade pip~=21.2.0
 
 COPY ./dockerfiles/mlrun/requirements.txt ./mlrun-image-requirements.txt
 COPY ./extras-requirements.txt ./extras-requirements.txt

dockerfiles/models-gpu/Dockerfile

@@ -49,7 +49,7 @@ RUN conda config --add channels conda-forge && \
     conda update -n base -c defaults conda && \
     conda install -n base \
         python=3.7 \
-        pip~=20.2.0 \
+        pip~=21.2.0 \
     && conda clean -aqy
 
 RUN conda install -n base cmake cython \
@@ -94,16 +94,24 @@ WORKDIR /mlrun
 COPY ./requirements.txt ./
 COPY ./extras-requirements.txt ./extras-requirements.txt
 COPY ./dockerfiles/mlrun/requirements.txt ./mlrun-image-requirements.txt
-COPY ./dockerfiles/base/requirements.txt ./base-image-requirements.txt
-COPY ./dockerfiles/models/requirements.txt ./models-image-requirements.txt
-COPY ./dockerfiles/models-gpu/requirements.txt ./models-gpu-image-requirements.txt
 
 RUN python -m pip install \
     -r requirements.txt \
     -r extras-requirements.txt \
-    -r mlrun-image-requirements.txt \
-    -r base-image-requirements.txt \
-    -r models-image-requirements.txt \
+    -r mlrun-image-requirements.txt
+
+# if we give all the requirements together pip resolver go nuts so separating between the package and the image
+# requirements
+COPY ./dockerfiles/base/requirements.txt ./base-image-requirements.txt
+RUN python -m pip install \
+    -r base-image-requirements.txt
+
+COPY ./dockerfiles/models/requirements.txt ./models-image-requirements.txt
+RUN python -m pip install \
+    -r models-image-requirements.txt
+
+COPY ./dockerfiles/models-gpu/requirements.txt ./models-gpu-image-requirements.txt
+RUN python -m pip install \
     -r models-gpu-image-requirements.txt
 
 COPY . .

dockerfiles/models-gpu/py36/Dockerfile

@@ -36,7 +36,7 @@ RUN conda config --add channels conda-forge && \
     conda update -n base -c defaults conda && \
     conda install -n base \
         python=3.6 \
-        pip~=20.2.0 \
+        pip~=21.2.0 \
     && conda clean -aqy
 
 RUN conda install -n base cmake cython \
@@ -79,16 +79,24 @@ WORKDIR /mlrun
 COPY ./requirements.txt ./
 COPY ./extras-requirements.txt ./extras-requirements.txt
 COPY ./dockerfiles/mlrun/requirements.txt ./mlrun-image-requirements.txt
-COPY ./dockerfiles/base/requirements.txt ./base-image-requirements.txt
-COPY ./dockerfiles/models/requirements.txt ./models-image-requirements.txt
-COPY ./dockerfiles/models-gpu/requirements.txt ./models-gpu-image-requirements.txt
 
 RUN python -m pip install \
     -r requirements.txt \
     -r extras-requirements.txt \
-    -r mlrun-image-requirements.txt \
-    -r base-image-requirements.txt \
-    -r models-image-requirements.txt \
+    -r mlrun-image-requirements.txt
+
+# if we give all the requirements together pip resolver go nuts so separating between the package and the image
+# requirements
+COPY ./dockerfiles/base/requirements.txt ./base-image-requirements.txt
+RUN python -m pip install \
+    -r base-image-requirements.txt
+
+COPY ./dockerfiles/models/requirements.txt ./models-image-requirements.txt
+RUN python -m pip install \
+    -r models-image-requirements.txt
+
+COPY ./dockerfiles/models-gpu/requirements.txt ./models-gpu-image-requirements.txt
+RUN python -m pip install \
     -r models-gpu-image-requirements.txt
 
 COPY . .

dockerfiles/test-system/Dockerfile

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG MLRUN_PYTHON_VERSION=3.7.9
+ARG MLRUN_PYTHON_VERSION=3.7.11
 
 FROM quay.io/mlrun/python:${MLRUN_PYTHON_VERSION}-slim
 
@@ -24,7 +24,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   make \
  && rm -rf /var/lib/apt/lists/*
 
-RUN python -m pip install --upgrade pip~=20.2.0
+RUN python -m pip install --upgrade pip~=21.2.0
 
 COPY ./dockerfiles/test-system/requirements.txt /tmp/mlrun-requirements/test-system-requirements.txt
 COPY ./dockerfiles/mlrun-api/requirements.txt /tmp/mlrun-requirements/mlrun-api-requirements.txt

dockerfiles/test/Dockerfile

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG MLRUN_PYTHON_VERSION=3.7
+ARG MLRUN_PYTHON_VERSION=3.7.11
 
 FROM quay.io/mlrun/python:${MLRUN_PYTHON_VERSION}-slim
 
@@ -39,7 +39,7 @@ RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debia
 RUN apt update && apt-get install -y docker-ce-cli \
     && rm -rf /var/lib/apt/lists/*
 
-RUN python -m pip install --upgrade pip~=20.2.0
+RUN python -m pip install --upgrade pip~=21.2.0
 
 WORKDIR /mlrun
 
@@ -52,8 +52,16 @@ COPY ./docs/requirements.txt ./docs-requirements.txt
 RUN pip install \
     -r requirements.txt \
     -r extras-requirements.txt \
-    -r mlrun-api-requirements.txt \
-    -r dev-requirements.txt \
+    -r mlrun-api-requirements.txt
+
+# if we give all the requirements together pip resolver go nuts so separating between the package and the image
+# requirements
+COPY ./dev-requirements.txt ./
+RUN python -m pip install \
+    -r dev-requirements.txt
+
+COPY ./docs/requirements.txt ./docs-requirements.txt
+RUN python -m pip install \
     -r docs-requirements.txt
 
 COPY . .

mlrun/builder.py

@@ -289,7 +289,7 @@ def resolve_mlrun_install_command(mlrun_version_specifier=None):
     if not mlrun_version_specifier:
         if config.httpdb.builder.mlrun_version_specifier:
             mlrun_version_specifier = config.httpdb.builder.mlrun_version_specifier
-        elif config.version == "unstable":
+        elif config.version in ["unstable", "0.0.0+unstable"]:
             mlrun_version_specifier = (
                 f"{config.package_path}[complete] @ git+"
                 f"https://github.com/mlrun/mlrun@development"

mlrun/db/httpdb.py

@@ -2249,6 +2249,15 @@ def _validate_version_compatibility(server_version, client_version) -> bool:
                 client_version=client_version,
             )
             return True
+        if (parsed_server_version.major == 0 and parsed_server_version.minor == 0) or (
+            parsed_client_version.major == 0 and parsed_client_version.minor == 0
+        ):
+            logger.warning(
+                "Server or client version is unstable. Assuming compatible",
+                server_version=server_version,
+                client_version=client_version,
+            )
+            return True
         if parsed_server_version.major != parsed_client_version.major:
             logger.warning(
                 "Server and client versions are incompatible",

mlrun/utils/helpers.py

@@ -563,7 +563,9 @@ def _set_artifact_path(task):
 
 def enrich_image_url(image_url: str) -> str:
     image_url = image_url.strip()
-    tag = config.images_tag or mlrun.utils.version.Version().get()["version"]
+    tag = config.images_tag or mlrun.utils.version.Version().get()["version"].replace(
+        "+", "-"
+    )
     registry = config.images_registry
 
     # it's an mlrun image if the repository is mlrun

mlrun/utils/version/version.py

@@ -14,7 +14,7 @@ class Version(metaclass=Singleton):
     def __init__(self):
         # When installing un-released version (e.g. by doing pip install git+https://github.com/mlrun/mlrun@development)
         # it won't have a version file, so adding some sane defaults
-        self.version_info = {"git_commit": "unknown", "version": "unstable"}
+        self.version_info = {"git_commit": "unknown", "version": "0.0.0+unstable"}
         try:
             self.version_info = json.loads(
                 read_text("mlrun.utils.version", "version.json")

setup.py

@@ -32,8 +32,8 @@ def version():
         # When installing un-released version (e.g. by doing
         # pip install git+https://github.com/mlrun/mlrun@development)
         # it won't have a version file, so adding some sane default
-        logger.warning("Failed resolving version. Ignoring and using unstable")
-        return "unstable"
+        logger.warning("Failed resolving version. Ignoring and using 0.0.0+unstable")
+        return "0.0.0+unstable"
 
 
 def is_ignored(line):