[Requirements] Support extras (#620)

.github/workflows/ci.yaml

@@ -76,6 +76,28 @@ jobs:
       - name: Run Dockerized DB Migration tests
         run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-migrations-dockerized
 
+  package-tests:
+    name: Run package tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python-version }}-
+          ${{ runner.os }}-pip-
+    - name: Test package
+      run: MLRUN_PYTHON_VERSION=${{ matrix.python-version }} make test-package
+
   docs:
     name: Build Project Documentation
     runs-on: ubuntu-latest

Makefile

@@ -423,6 +423,10 @@ test-migrations: clean ## Run mlrun db migrations tests
 test-system: build-test-system ## Run mlrun system tests
 	docker run -t --rm $(MLRUN_SYSTEM_TEST_IMAGE_NAME)
 
+.PHONY: test-package
+test-package: ## Run mlrun package tests
+	PYTHON_VERSION=$(MLRUN_PYTHON_VERSION) bash ./automation/package_test/test_imports.sh
+
 .PHONY: run-api-undockerized
 run-api-undockerized: ## Run mlrun api locally (un-dockerized)
 	python -m mlrun db

automation/package_test/test_imports.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -o errexit
+
+
+test_import () {
+    echo "Testing import: python=$PYTHON_VERSION extra=$1 import=$2"
+    # Create an empty environment
+    python -m pip install virtualenv
+    virtualenv venv
+    source venv/bin/activate
+    pip install ."$1"
+    python -c "$2"
+    deactivate
+    rm -rf venv
+}
+
+basic_test="import mlrun"
+api_test="import mlrun.api.main"
+s3_test="import mlrun.datastore.s3"
+azure_blob_storage_test="import mlrun.datastore.azure_blob"
+
+test_import ""                      "$basic_test"
+# API works only with python 3.7 and above
+if [ "$PYTHON_VERSION" != "3.6" ]
+  then
+    test_import "[api]"                 "$basic_test; $api_test"
+    test_import "[complete-api]"        "$basic_test; $api_test; $s3_test; $azure_blob_storage_test"
+fi
+test_import "[s3]"                  "$basic_test; $s3_test"
+test_import "[azure-blob-storage]"  "$basic_test; $azure_blob_storage_test"
+test_import "[complete]"  "$basic_test; $s3_test; $azure_blob_storage_test"

automation/system_test/prepare.py

@@ -279,9 +279,9 @@ def _prepare_test_env(self):
 
     def _override_mlrun_api_env(self):
         version_specifier = (
-            f"git+https://github.com/mlrun/mlrun@{self._mlrun_commit}"
+            f"mlrun[complete] @ git+https://github.com/mlrun/mlrun@{self._mlrun_commit}"
             if self._mlrun_commit
-            else "mlrun"
+            else "mlrun[complete]"
         )
         data = {
             "MLRUN_HTTPDB__BUILDER__MLRUN_VERSION_SPECIFIER": version_specifier,

dockerfiles/base/Dockerfile

@@ -31,7 +31,7 @@ COPY ./dockerfiles/base/requirements.txt ./
 RUN python -m pip install -r requirements.txt
 
 COPY . .
-RUN python -m pip install .
+RUN python -m pip install .[complete]
 
 ARG MLRUN_MLUTILS_GITHUB_TAG=development
 ARG MLRUN_MLUTILS_CACHE_DATE=initial

dockerfiles/jupyter/Dockerfile

@@ -23,7 +23,7 @@ COPY ./dockerfiles/jupyter/requirements.txt /tmp
 RUN python -m pip install -r /tmp/requirements.txt && rm -f /tmp/requirements.txt
 
 COPY . /tmp/mlrun
-RUN cd /tmp/mlrun && python -m pip install ".[api]" && cd /tmp && rm -rf mlrun
+RUN cd /tmp/mlrun && python -m pip install ".[complete]" && cd /tmp && rm -rf mlrun
 
 RUN chown -R $NB_UID:$NB_GID $HOME
 

dockerfiles/mlrun-api/Dockerfile

@@ -43,6 +43,8 @@ RUN python -m pip install \
 ENV MLRUN_httpdb__dirpath=/mlrun/db
 ENV MLRUN_httpdb__port=8080
 COPY . .
+
+RUN python -m pip install .[complete-api]
 VOLUME /mlrun/db
 
 CMD ["python",  "-m",  "mlrun",  "db"]

dockerfiles/mlrun/Dockerfile

@@ -33,4 +33,4 @@ RUN python -m pip install \
     -r mlrun-image-requirements.txt
 
 COPY . .
-RUN python -m pip install .
+RUN python -m pip install .[complete]

dockerfiles/models-gpu/Dockerfile

@@ -144,7 +144,7 @@ COPY ./dockerfiles/models-gpu/requirements.txt ./
 RUN python -m pip install -r requirements.txt
 
 COPY . .
-RUN python -m pip install .
+RUN python -m pip install .[complete]
 
 ARG MLRUN_MLUTILS_GITHUB_TAG=development
 ARG MLRUN_MLUTILS_CACHE_DATE=initial

dockerfiles/models-gpu/py36/Dockerfile

@@ -137,7 +137,7 @@ COPY ./dockerfiles/models-gpu/requirements.txt ./
 RUN python -m pip install -r requirements.txt
 
 COPY . .
-RUN python -m pip install .
+RUN python -m pip install .[complete]
 
 ARG MLRUN_MLUTILS_GITHUB_TAG=development
 ARG MLRUN_MLUTILS_CACHE_DATE=initial

dockerfiles/models/Dockerfile

@@ -73,7 +73,7 @@ COPY ./dockerfiles/models/requirements.txt ./
 RUN python -m pip install -r requirements.txt
 
 COPY . .
-RUN python -m pip install .
+RUN python -m pip install .[complete]
 
 ARG MLRUN_MLUTILS_GITHUB_TAG=development
 ARG MLRUN_MLUTILS_CACHE_DATE=initial

dockerfiles/models/py36/Dockerfile

@@ -76,7 +76,7 @@ COPY ./dockerfiles/models/requirements.txt ./
 RUN python -m pip install -r requirements.txt
 
 COPY . .
-RUN python -m pip install .
+RUN python -m pip install .[complete]
 
 ARG MLRUN_MLUTILS_GITHUB_TAG=development
 ARG MLRUN_MLUTILS_CACHE_DATE=initial

dockerfiles/test-system/Dockerfile

@@ -27,6 +27,6 @@ RUN python -m pip install --upgrade pip
 COPY ./dockerfiles/test-system/requirements.txt /tmp
 RUN python -m pip install -r /tmp/requirements.txt && rm -f /tmp/requirements.txt
 COPY . /tmp/mlrun
-RUN cd /tmp/mlrun && python -m pip install ".[api]" && mv tests /tests && cd /tmp && rm -rf mlrun
+RUN cd /tmp/mlrun && python -m pip install ".[complete]" && mv tests /tests && cd /tmp && rm -rf mlrun
 
 CMD ["pytest",  "--capture=no", "-rf",  "-v",  "--disable-warnings", "/tests/system"]

dockerfiles/test/Dockerfile

@@ -53,4 +53,4 @@ RUN pip install \
 
 COPY . .
 
-RUN pip install -e .
+RUN pip install -e .[complete]

mlrun/builder.py

@@ -237,9 +237,14 @@ def _resolve_mlrun_install_command(mlrun_version_specifier):
         if config.httpdb.builder.mlrun_version_specifier:
             mlrun_version_specifier = config.httpdb.builder.mlrun_version_specifier
         elif config.version == "unstable":
-            mlrun_version_specifier = "git+https://github.com/mlrun/mlrun@development"
+            mlrun_version_specifier = (
+                f"{config.package_path}[complete] @ git+"
+                f"https://github.com/mlrun/mlrun@development"
+            )
         else:
-            mlrun_version_specifier = f"{config.package_path}=={config.version}"
+            mlrun_version_specifier = (
+                f"{config.package_path}[complete]=={config.version}"
+            )
     return f"pip install {mlrun_version_specifier}"
 
 

mlrun/datastore/datastore.py

@@ -15,11 +15,10 @@
 from urllib.parse import urlparse
 
 import mlrun
-from .azure_blob import AzureBlobStore
+import mlrun.errors
 from .base import DataItem, HttpStore
 from .filestore import FileStore
 from .inmem import InMemoryStore
-from .s3 import S3Store
 from .v3io import V3ioStore
 from ..config import config
 from ..utils import run_keys, DB_SCHEMA
@@ -48,11 +47,26 @@ def parse_url(url):
 
 
 def schema_to_store(schema):
+    # import store classes inside to enable making their dependencies optional (package extras)
     if not schema or schema in ["file", "c", "d"]:
         return FileStore
     elif schema == "s3":
+        try:
+            from .s3 import S3Store
+        except ImportError:
+            raise mlrun.errors.MLRunMissingDependencyError(
+                "s3 packages are missing, use pip install mlrun[s3]"
+            )
+
         return S3Store
     elif schema == "az":
+        try:
+            from .azure_blob import AzureBlobStore
+        except ImportError:
+            raise mlrun.errors.MLRunMissingDependencyError(
+                "azure blob storage packages are missing, use pip install mlrun[azure-blob-storage]"
+            )
+
         return AzureBlobStore
     elif schema in ["v3io", "v3ios"]:
         return V3ioStore

mlrun/errors.py

@@ -92,11 +92,20 @@ class MLRunIncompatibleVersionError(MLRunHTTPStatusError):
     error_status_code = HTTPStatus.BAD_REQUEST.value
 
 
+class MLRunInternalServerError(MLRunHTTPStatusError):
+    error_status_code = HTTPStatus.INTERNAL_SERVER_ERROR.value
+
+
+class MLRunMissingDependencyError(MLRunInternalServerError):
+    pass
+
+
 STATUS_ERRORS = {
     HTTPStatus.BAD_REQUEST.value: MLRunBadRequestError,
     HTTPStatus.UNAUTHORIZED.value: MLRunUnauthorizedError,
     HTTPStatus.FORBIDDEN.value: MLRunAccessDeniedError,
     HTTPStatus.NOT_FOUND.value: MLRunNotFoundError,
     HTTPStatus.CONFLICT.value: MLRunConflictError,
     HTTPStatus.PRECONDITION_FAILED.value: MLRunPreconditionFailedError,
+    HTTPStatus.INTERNAL_SERVER_ERROR.value: MLRunInternalServerError,
 }

requirements.txt

@@ -1,9 +1,9 @@
 # >=1.25.4, <1.27 from botocore 1.19.28 inside boto3 1.16.28 inside nuclio-jupyter 0.8.8
 urllib3>=1.25.4, <1.27
+# >=3.0.2 from requests 2.25.1 <4.0 from aiohttp 3.7.3, requests is <5, so without the upbound there's a conflict
+chardet>=3.0.2, <4.0
 GitPython~=3.0
-# required by google-auth - TODO: check if still needed
 aiohttp~=3.6
-boto3~=1.9
 click~=7.0
 # 3.0 iguazio system uses 1.0.1, since the sdk is still mark as beta (and not stable) I'm limiting to only patch changes
 kfp~=1.0.1
@@ -19,9 +19,6 @@ requests~=2.22
 sqlalchemy~=1.3
 tabulate>=0.8.0, <=0.8.3
 v3io~=0.5.0
-# required by some sub-dependency of a package installed in models-gpu, otherwise building this image fails - TODO: check if still happening
-google-auth<2.0dev,>=1.19.1
-azure-storage-blob~=12.0
 pydantic~=1.5
 # <3.4 since 3.4 can't be installed on pip 18.1
 orjson>=3,<3.4
@@ -35,6 +32,6 @@ dask~=2.12
 # 3.0 iguazio system is running k8s 1.17 so ideally we would use 17.X, but kfp limiting to <12.0
 kubernetes~=11.0
 # TODO: move to API requirements (shouldn't really be here, the sql run db using the API sqldb is preventing us from
-#  separating the SDK and API code) (referring to humanfriendly and fastapi
+#  separating the SDK and API code) (referring to humanfriendly and fastapi)
 humanfriendly~=8.2
 fastapi~=0.62.0

setup.py

@@ -54,6 +54,24 @@ def load_deps(path):
 tests_require = list(load_deps("dev-requirements.txt"))
 api_deps = list(load_deps("dockerfiles/mlrun-api/requirements.txt"))
 
+# NOTE: These are tested in `automation/package_test/test_imports.sh` If
+# you modify these, make sure to change the corresponding line there.
+extras_require = {
+    "s3": ["boto3~=1.9"],
+    "azure-blob-storage": ["azure-storage-blob~=12.0"],
+}
+extras_require["complete"] = sorted(
+    {
+        requirement
+        for requirement_list in extras_require.values()
+        for requirement in requirement_list
+    }
+)
+extras_require["api"] = api_deps
+complete_api_deps = set(api_deps)
+complete_api_deps.update(extras_require["complete"])
+extras_require["complete-api"] = sorted(complete_api_deps)
+
 
 setup(
     name="mlrun",
@@ -93,7 +111,7 @@ def load_deps(path):
         "mlrun.api.utils.projects.remotes",
     ],
     install_requires=install_requires,
-    extras_require={"api": api_deps},
+    extras_require=extras_require,
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",

tests/Dockerfile.test-nb

@@ -27,4 +27,4 @@ RUN pip install \
     -r dev-requirements.txt
 COPY . .
 ENV PYTHONPATH=/mlrun
-RUN python -m pip install .
+RUN python -m pip install .[complete]