feat(service): backport API versioning changes from 1.0.0 release

.github/workflows/acceptance-tests.yml

@@ -27,6 +27,7 @@ jobs:
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
+      extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
       - id: deploy-comment
         uses: SwissDataScienceCenter/renku-actions/check-pr-description@v0.1.0
@@ -63,6 +64,7 @@ jobs:
         renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
         renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
         renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}"
+        extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
     - name: Check existing renkubot comment
       uses: peter-evans/find-comment@v1
       id: findcomment

.github/workflows/test_deploy.yml

@@ -291,7 +291,9 @@ jobs:
     - name: Install dependencies
       run: |
         brew update || true
-        brew install git-lfs shellcheck node || brew link --overwrite node
+        brew unlink git-lfs || true
+        curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
+        brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
         python -m pip install setuptools wheel twine
         python -m pip install -e .[all]
@@ -322,7 +324,9 @@ jobs:
     - name: Install dependencies
       run: |
         brew update || true
-        brew install git-lfs shellcheck node || brew link --overwrite node
+        brew unlink git-lfs || true
+        curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
+        brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
         python -m pip install setuptools wheel twine
         python -m pip install -e .[all]
@@ -359,7 +363,9 @@ jobs:
       - name: Install dependencies
         run: |
           brew update || true
-          brew install git-lfs shellcheck node || brew link --overwrite node
+          brew unlink git-lfs || true
+          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
+          brew install shellcheck node || brew link --overwrite node
           python -m pip install --upgrade pip
           python -m pip install setuptools wheel twine
           python -m pip install -e .[all]
@@ -396,7 +402,9 @@ jobs:
       - name: Install dependencies
         run: |
           brew update || true
-          brew install git-lfs shellcheck node || brew link --overwrite node
+          brew unlink git-lfs || true
+          curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
+          brew install shellcheck node || brew link --overwrite node
           python -m pip install --upgrade pip
           python -m pip install setuptools wheel twine
           python -m pip install -e .[all]
@@ -483,7 +491,9 @@ jobs:
     - name: Install dependencies
       run: |
         brew update || true
-        brew install git-lfs shellcheck node || brew link --overwrite node
+        brew unlink git-lfs || true
+        curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
+        brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
         python -m pip install setuptools wheel
         python -m pip install -e .[all]

.github/workflows/test_publish.yml

@@ -56,7 +56,8 @@ jobs:
     - name: Install dependencies
       run: |
         brew update
-        brew install git-lfs shellcheck node || brew link --overwrite node
+        curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
+        brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
         python -m pip install -e .[all]
         git config --global --add user.name "Renku @ SDSC"

renku/core/commands/providers/dataverse.py

@@ -34,6 +34,7 @@
 from renku.core.commands.providers.doi import DOIProvider
 from renku.core.models.datasets import Dataset, DatasetFile, DatasetSchema
 from renku.core.models.provenance.agents import PersonSchema
+from renku.core.utils import communication
 from renku.core.utils.doi import extract_doi, is_doi
 from renku.core.utils.requests import retry
 
@@ -48,6 +49,24 @@
 DATAVERSE_EXPORTER = "schema.org"
 
 
+DATAVERSE_SUBJECTS = [
+    "Agricultural Sciences",
+    "Arts and Humanities",
+    "Astronomy and Astrophysics",
+    "Business and Management",
+    "Chemistry",
+    "Computer and Information Science",
+    "Earth and Environmental Sciences",
+    "Engineering",
+    "Law",
+    "Mathematical Sciences",
+    "Medicine, Health and Life Sciences",
+    "Physics",
+    "Social Sciences",
+    "Other",
+]
+
+
 class _DataverseDatasetSchema(DatasetSchema):
     """Schema for Dataverse datasets."""
 
@@ -411,15 +430,26 @@ def export(self, publish, **kwargs):
 
     def _get_dataset_metadata(self):
         authors, contacts = self._get_creators()
+        subject = self._get_subject()
         metadata_template = Template(DATASET_METADATA_TEMPLATE)
         metadata = metadata_template.substitute(
             name=_escape_json_string(self.dataset.title),
             authors=json.dumps(authors),
             contacts=json.dumps(contacts),
             description=_escape_json_string(self.dataset.description),
+            subject=subject,
         )
         return json.loads(metadata)
 
+    def _get_subject(self):
+        text_prompt = "Subject of this dataset: \n\n"
+        text_prompt += "\n".join(f"{s}\t[{i}]" for i, s in enumerate(DATAVERSE_SUBJECTS, start=1))
+        text_prompt += "\n\nSubject"
+
+        selection = communication.prompt(text_prompt, type=int, default=len(DATAVERSE_SUBJECTS)) or 0
+
+        return DATAVERSE_SUBJECTS[selection - 1]
+
     def _get_creators(self):
         authors = []
         contacts = []

renku/core/commands/providers/dataverse_metadata_templates.py

@@ -56,7 +56,7 @@
                         "typeName": "dsDescription"
                     },
                     {
-                        "value": [],
+                        "value": ["${subject}"],
                         "typeClass": "controlledVocabulary",
                         "multiple": true,
                         "typeName": "subject"

renku/service/controllers/cache_migrations_check.py

@@ -78,4 +78,4 @@ def renku_op(self):
 
     def to_response(self):
         """Execute controller flow and serialize to service response."""
-        return result_response(MigrationsCheckCtrl.RESPONSE_SERIALIZER, self.execute_op())
+        return result_response(self.RESPONSE_SERIALIZER, self.execute_op())

renku/service/controllers/utils/project_clone.py

@@ -40,7 +40,11 @@ def user_project_clone(cache, user_data, project_data):
                 project.abs_path,
                 depth=project_data["depth"],
                 raise_git_except=True,
-                config={"user.name": project_data["fullname"], "user.email": project_data["email"]},
+                config={
+                    "user.name": project_data["fullname"],
+                    "user.email": project_data["email"],
+                    "pull.rebase": False,
+                },
                 checkout_rev=project_data["ref"],
             )
         ).output

renku/service/controllers/version.py

@@ -28,9 +28,14 @@ class VersionCtrl(ServiceCtrl):
 
     RESPONSE_SERIALIZER = VersionResponseRPC()
 
-    def to_response(self):
+    def to_response(self, minimum_version, maximum_version):
         """Serialize to service version response."""
         return result_response(
             VersionCtrl.RESPONSE_SERIALIZER,
-            {"latest_version": __version__, "supported_project_version": SUPPORTED_PROJECT_VERSION},
+            {
+                "latest_version": __version__,
+                "supported_project_version": SUPPORTED_PROJECT_VERSION,
+                "minimum_api_version": minimum_version.name,
+                "maximum_api_version": maximum_version.name,
+            },
         )

renku/service/serializers/version.py

@@ -22,6 +22,8 @@ class VersionResponse(Schema):
 
     latest_version = fields.String()
     supported_project_version = fields.Number()
+    minimum_api_version = fields.String()
+    maximum_api_version = fields.String()
 
 
 class VersionResponseRPC(Schema):

renku/service/views/api_versions.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service api versions."""
+from typing import Any, Callable, List, Optional
+
+from flask import Blueprint
+
+
+class ApiVersion:
+    """Represents a Blueprint API version."""
+
+    def __init__(self, name: str, is_base_version: bool = False):
+        self.name = name
+        self.is_base_version = is_base_version
+
+
+class VersionedBlueprint(Blueprint):
+    """A Blueprint that supports versioning."""
+
+    def add_url_rule(
+        self,
+        rule: str,
+        endpoint: Optional[str] = None,
+        view_func: Optional[Callable] = None,
+        provide_automatic_options: Optional[bool] = None,
+        versions: List[str] = None,
+        **options: Any,
+    ) -> None:
+        """Overwrite Blueprint add_url_rule to support versioning."""
+
+        for version in versions:
+            if version.is_base_version:
+                super().add_url_rule(
+                    rule, endpoint, view_func, provide_automatic_options=provide_automatic_options, **options
+                )
+
+            version_rule = f"/{version.name}{rule}"
+            super().add_url_rule(
+                version_rule, endpoint, view_func, provide_automatic_options=provide_automatic_options, **options
+            )
+
+
+V0_9 = ApiVersion("0.9", is_base_version=True)
+
+MINIMUM_VERSION = V0_9
+MAXIMUM_VERSION = V0_9

renku/service/views/apispec.py

@@ -16,10 +16,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Renku service apispec views."""
-from apispec import APISpec
+from apispec import APISpec, yaml_utils
 from apispec.ext.marshmallow import MarshmallowPlugin
 from apispec_webframeworks.flask import FlaskPlugin
 from flask import Blueprint, current_app, jsonify
+from flask.views import MethodView
 
 from renku.service.config import (
     API_VERSION,
@@ -85,11 +86,28 @@
 
 """
 
+
+class MultiURLFlaskPlugin(FlaskPlugin):
+    """FlaskPlugin extension that supports multiple URLs per endpoint."""
+
+    def path_helper(self, path, operations, *, view, app=None, **kwargs):
+        """Path helper that allows passing a Flask view function."""
+        rule = self._rule_for_view(view, app=app)
+        operations.update(yaml_utils.load_operations_from_docstring(view.__doc__))
+        if hasattr(view, "view_class") and issubclass(view.view_class, MethodView):
+            for method in view.methods:
+                if method in rule.methods:
+                    method_name = method.lower()
+                    method = getattr(view.view_class, method_name)
+                    operations[method_name] = yaml_utils.load_yaml_from_docstring(method.__doc__)
+        return self.flaskpath2openapi(path)
+
+
 spec = APISpec(
     title=SERVICE_NAME,
     openapi_version=OPENAPI_VERSION,
     version=API_VERSION,
-    plugins=[FlaskPlugin(), MarshmallowPlugin()],
+    plugins=[MultiURLFlaskPlugin(), MarshmallowPlugin()],
     servers=[{"url": SERVICE_API_BASE_PATH}],
     security=[{"oidc": []}, {"JWT": [], "gitlab-token": []}],
     info={"description": TOP_LEVEL_DESCRIPTION},
@@ -109,5 +127,5 @@ def openapi():
 def get_apispec(app):
     """Return the apispec."""
     for rule in current_app.url_map.iter_rules():
-        spec.path(view=app.view_functions[rule.endpoint])
+        spec.path(path=rule.rule, view=app.view_functions[rule.endpoint])
     return spec

renku/service/views/cache.py

@@ -16,7 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Renku service cache views."""
-from flask import Blueprint, request
+from flask import request
 
 from renku.service.config import SERVICE_PREFIX
 from renku.service.controllers.cache_files_upload import UploadFilesCtrl
@@ -25,6 +25,7 @@
 from renku.service.controllers.cache_migrate_project import MigrateProjectCtrl
 from renku.service.controllers.cache_migrations_check import MigrationsCheckCtrl
 from renku.service.controllers.cache_project_clone import ProjectCloneCtrl
+from renku.service.views.api_versions import V0_9, VersionedBlueprint
 from renku.service.views.decorators import (
     accepts_json,
     handle_common_except,
@@ -34,10 +35,10 @@
 )
 
 CACHE_BLUEPRINT_TAG = "cache"
-cache_blueprint = Blueprint("cache", __name__, url_prefix=SERVICE_PREFIX)
+cache_blueprint = VersionedBlueprint("cache", __name__, url_prefix=SERVICE_PREFIX)
 
 
-@cache_blueprint.route("/cache.files_list", methods=["GET"], provide_automatic_options=False)
+@cache_blueprint.route("/cache.files_list", methods=["GET"], provide_automatic_options=False, versions=[V0_9])
 @handle_common_except
 @requires_cache
 @requires_identity
@@ -60,7 +61,7 @@ def list_uploaded_files_view(user_data, cache):
     return ListUploadedFilesCtrl(cache, user_data).to_response()
 
 
-@cache_blueprint.route("/cache.files_upload", methods=["POST"], provide_automatic_options=False)
+@cache_blueprint.route("/cache.files_upload", methods=["POST"], provide_automatic_options=False, versions=[V0_9])
 @handle_common_except
 @requires_cache
 @requires_identity
@@ -95,7 +96,7 @@ def upload_file_view(user_data, cache):
     return UploadFilesCtrl(cache, user_data, request).to_response()
 
 
-@cache_blueprint.route("/cache.project_clone", methods=["POST"], provide_automatic_options=False)
+@cache_blueprint.route("/cache.project_clone", methods=["POST"], provide_automatic_options=False, versions=[V0_9])
 @handle_common_except
 @accepts_json
 @requires_cache
@@ -124,7 +125,7 @@ def project_clone_view(user_data, cache):
     return ProjectCloneCtrl(cache, user_data, dict(request.json)).to_response()
 
 
-@cache_blueprint.route("/cache.project_list", methods=["GET"], provide_automatic_options=False)
+@cache_blueprint.route("/cache.project_list", methods=["GET"], provide_automatic_options=False, versions=[V0_9])
 @handle_common_except
 @requires_cache
 @requires_identity
@@ -147,7 +148,7 @@ def list_projects_view(user_data, cache):
     return ListProjectsCtrl(cache, user_data).to_response()
 
 
-@cache_blueprint.route("/cache.migrate", methods=["POST"], provide_automatic_options=False)
+@cache_blueprint.route("/cache.migrate", methods=["POST"], provide_automatic_options=False, versions=[V0_9])
 @handle_common_except
 @handle_migration_except
 @accepts_json
@@ -176,7 +177,7 @@ def migrate_project_view(user_data, cache):
     return MigrateProjectCtrl(cache, user_data, dict(request.json)).to_response()
 
 
-@cache_blueprint.route("/cache.migrations_check", methods=["GET"], provide_automatic_options=False)
+@cache_blueprint.route("/cache.migrations_check", methods=["GET"], provide_automatic_options=False, versions=[V0_9])
 @handle_common_except
 @requires_cache
 @requires_identity