diff --git a/Makefile b/Makefile index 158056a500..903c2a3ee9 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ DOCKER_PREFIX:=${DOCKER_REGISTRY}$(DOCKER_REPOSITORY) GIT_MASTER_HEAD_SHA:=$(shell git rev-parse --short --verify HEAD) TEMPLATE_URL:=https://github.com/SwissDataScienceCenter/renku-project-template -TEMPLATE_REFERENCE:=0.3.0 +TEMPLATE_REFERENCE:=$(shell sed -n -E 's/^__template_version__ = "([^"]+)"/\1/p' renku/version.py) TEMPLATE_DIR:=renku/templates/ .PHONY: service cli docker-tag docker-push docker-login @@ -47,6 +47,7 @@ cli: docker build -f Dockerfile -t $(DOCKER_PREFIX)renku-python:`git rev-parse --short HEAD` --build-arg CLEAN_INSTALL=1 . download-templates: + @[ "${TEMPLATE_REFERENCE}" ] || ( echo "__template_version__ is not set"; exit 1 ) @echo "Downloading templates" rm -rf $(TEMPLATE_DIR) mkdir -p $(TEMPLATE_DIR) diff --git a/renku/__init__.py b/renku/__init__.py index 8d7c626be1..e3168beabb 100644 --- a/renku/__init__.py +++ b/renku/__init__.py @@ -19,6 +19,6 @@ from __future__ import absolute_import, print_function -from renku.version import __version__ +from renku.version import __template_version__, __version__ -__all__ = ("__version__",) +__all__ = ("__template_version__", "__version__") diff --git a/renku/cli/dataset.py b/renku/cli/dataset.py index 65fb8e96d9..d46b8d6ad0 100644 --- a/renku/cli/dataset.py +++ b/renku/cli/dataset.py @@ -1000,7 +1000,7 @@ def update(names, creators, include, exclude, ref, delete, external, no_external def get_dataset_files(records): from renku.core.commands.format.tabulate import tabulate - columns = {"path": ("path", None), "dataset": ("dataset.name", None), "external": ("external", None)} + columns = {"path": ("path", None), "dataset": ("dataset.name", "dataset"), "external": ("external", None)} return tabulate(collection=records, columns="path,dataset,external", columns_mapping=columns) if not datasets and not dataset_files: diff --git a/renku/core/commands/view_model/template.py b/renku/core/commands/view_model/template.py index 8d8d43501b..5c1a82fbe4 100644 --- a/renku/core/commands/view_model/template.py +++ b/renku/core/commands/view_model/template.py @@ -73,7 +73,7 @@ def from_template(cls, template: Template) -> "TemplateViewModel": parameters=template.parameters, icon=template.icon, immutable_files=template.immutable_files, - versions=template.get_all_versions(), + versions=template.get_all_references(), ) diff --git a/renku/core/management/template/template.py b/renku/core/management/template/template.py index ab601007de..ea1d41fc3f 100644 --- a/renku/core/management/template/template.py +++ b/renku/core/management/template/template.py @@ -354,48 +354,42 @@ def read_valid_value(var: TemplateParameter, default_value=None): class EmbeddedTemplates(TemplatesSource): """Represent templates that are bundled with Renku. - For embedded templates, ``source`` is "renku" and ``version`` is set to the installed Renku version. ``reference`` - should not be set for such projects. + For embedded templates, ``source`` is "renku". In the old versioning scheme, ``version`` is set to the installed + Renku version and ``reference`` is not set. In the new scheme, both ``version`` and ``reference`` are set to the + template version. """ @classmethod def fetch(cls, source: Optional[str], reference: Optional[str]) -> "EmbeddedTemplates": """Fetch embedded Renku templates.""" - from renku import __version__ - - if reference and reference != "master": - raise errors.ParameterError("Templates included in renku don't support specifying a template reference") + from renku import __template_version__ path = importlib_resources.files("renku") / "templates" with importlib_resources.as_file(path) as folder: path = Path(folder) - return cls(path=path, source="renku", reference=None, version=str(__version__)) - - def is_update_available(self, id: str, reference: Optional[str], version: Optional[str]) -> Tuple[bool, str]: - """Return True if an update is available along with the latest version of a template.""" - latest_version = self.get_latest_version(id=id, reference=reference, version=version) - update_available = latest_version is not None and latest_version != version - - return update_available, latest_version + return cls(path=path, source="renku", reference=__template_version__, version=__template_version__) - def get_all_versions(self, id) -> List[str]: - """Return all available versions for a template id.""" + def get_all_references(self, id) -> List[str]: + """Return all available references for a template id.""" template_exists = any(t.id == id for t in self.templates) - return [self.version] if template_exists else [] + return [self.reference] if template_exists else [] - def get_latest_version(self, id: str, reference: Optional[str], version: Optional[str]) -> Optional[str]: - """Return latest version number of a template.""" + def get_latest_reference_and_version( + self, id: str, reference: Optional[str], version: Optional[str] + ) -> Optional[Tuple[str, str]]: + """Return latest reference and version number of a template.""" if version is None: return + elif reference is None or reference != version: # Old versioning scheme + return self.reference, self.version - template_version = Version(self.version) try: current_version = Version(version) except ValueError: # NOTE: version is not a valid SemVer - return str(template_version) + return self.reference, self.version else: - return str(template_version) if current_version < template_version else version + return (self.reference, self.version) if current_version < Version(self.version) else (reference, version) def get_template(self, id, reference: Optional[str]) -> Optional["Template"]: """Return all available versions for a template id.""" @@ -432,14 +426,7 @@ def fetch(cls, source: Optional[str], reference: Optional[str]) -> "RepositoryTe return cls(path=path, source=source, reference=reference, version=version, repository=repository) - def is_update_available(self, id: str, reference: Optional[str], version: Optional[str]) -> Tuple[bool, str]: - """Return True if an update is available along with the latest version of a template.""" - latest_version = self.get_latest_version(id=id, reference=reference, version=version) - update_available = latest_version is not None and latest_version != reference - - return update_available, latest_version - - def get_all_versions(self, id) -> List[str]: + def get_all_references(self, id) -> List[str]: """Return a list of git tags that are valid SemVer and include a template id.""" versions = [] for tag in self.repository.tags: @@ -454,8 +441,10 @@ def get_all_versions(self, id) -> List[str]: return [str(v) for v in sorted(versions)] - def get_latest_version(self, id: str, reference: Optional[str], version: Optional[str]) -> Optional[str]: - """Return latest version number of a template.""" + def get_latest_reference_and_version( + self, id: str, reference: Optional[str], version: Optional[str] + ) -> Optional[Tuple[str, str]]: + """Return latest reference and version number of a template.""" if version is None: return @@ -463,11 +452,11 @@ def get_latest_version(self, id: str, reference: Optional[str], version: Optiona # NOTE: Assume that a SemVer reference is always a tag if tag: - versions = self.get_all_versions(id=id) - return versions[-1] if len(versions) > 0 else None + references = self.get_all_references(id=id) + return (references[-1], self.version) if len(references) > 0 else None # NOTE: Template's reference is a branch or SHA and the latest version is RepositoryTemplates' version - return self.version + return reference, self.version def _has_template_at(self, id: str, reference: str) -> bool: """Return if template id is available at a reference.""" diff --git a/renku/core/management/template/usecase.py b/renku/core/management/template/usecase.py index 92f5a1141a..42bc03ff15 100644 --- a/renku/core/management/template/usecase.py +++ b/renku/core/management/template/usecase.py @@ -70,13 +70,11 @@ def check_for_template_update(client) -> Tuple[bool, bool, Optional[str], Option metadata = TemplateMetadata.from_client(client=client) templates_source = fetch_templates_source(source=metadata.source, reference=metadata.reference) - latest_version = templates_source.get_latest_version( + update_available, latest_reference = templates_source.is_update_available( id=metadata.id, reference=metadata.reference, version=metadata.version ) - update_available = latest_version is not None and latest_version != metadata.version - - return update_available, metadata.allow_update, metadata.version, latest_version + return update_available, metadata.allow_update, metadata.reference, latest_reference @inject.autoparams("client_dispatcher") @@ -130,7 +128,7 @@ def update_template( templates_source = fetch_templates_source(source=template_metadata.source, reference=template_metadata.reference) - update_available, latest_version = templates_source.is_update_available( + update_available, latest_reference = templates_source.is_update_available( id=template_metadata.id, reference=template_metadata.reference, version=template_metadata.version ) @@ -139,7 +137,7 @@ def update_template( rendered_template, actions = _set_or_update_project_from_template( templates_source=templates_source, - reference=latest_version, + reference=latest_reference, id=template_metadata.id, interactive=interactive, dry_run=dry_run, diff --git a/renku/core/models/template.py b/renku/core/models/template.py index 309b826675..9226e8e52c 100644 --- a/renku/core/models/template.py +++ b/renku/core/models/template.py @@ -60,19 +60,26 @@ def templates(self) -> List["Template"]: return self.manifest.templates - @abstractmethod def is_update_available(self, id: str, reference: Optional[str], version: Optional[str]) -> Tuple[bool, str]: - """Return True if an update is available along with the latest version of a template.""" - raise NotImplementedError + """Return True if an update is available along with the latest reference of a template.""" + latest = self.get_latest_reference_and_version(id=id, reference=reference, version=version) + if not latest: + return False, reference + + latest_reference, latest_version = latest + update_available = latest_reference != reference or latest_version != version + return update_available, latest_reference @abstractmethod - def get_all_versions(self, id) -> List[str]: + def get_all_references(self, id) -> List[str]: """Return all available versions for a template id.""" raise NotImplementedError @abstractmethod - def get_latest_version(self, id: str, reference: Optional[str], version: Optional[str]) -> Optional[str]: - """Return latest version number of a template.""" + def get_latest_reference_and_version( + self, id: str, reference: Optional[str], version: Optional[str] + ) -> Optional[Tuple[str, str]]: + """Return latest reference and version number of a template.""" raise NotImplementedError @abstractmethod @@ -220,9 +227,9 @@ def templates_source(self, templates_source: TemplatesSource): self.version = templates_source.version self.path = templates_source.path / self.id - def get_all_versions(self) -> List[str]: - """Return all available versions for the template.""" - return self.templates_source.get_all_versions(self.id) + def get_all_references(self) -> List[str]: + """Return all available references for the template.""" + return self.templates_source.get_all_references(self.id) def validate(self, skip_files): """Validate a template.""" diff --git a/renku/version.py b/renku/version.py index f0e14c1893..7004259ffa 100644 --- a/renku/version.py +++ b/renku/version.py @@ -25,6 +25,7 @@ from importlib_metadata import distribution __version__ = "0.0.0" +__template_version__ = "0.3.0" def is_release(): diff --git a/tests/cli/test_template.py b/tests/cli/test_template.py index ad0926b7e1..bebd3d3ded 100644 --- a/tests/cli/test_template.py +++ b/tests/cli/test_template.py @@ -80,7 +80,7 @@ def test_template_show(isolated_runner): result = isolated_runner.invoke(cli, command + ["R-minimal"]) assert 0 == result.exit_code, format_result_exception(result) - assert "Name: Basic R (4.1.2) Project" in result.output + assert re.search("^Name: Basic R (.*) Project$", result.output, re.MULTILINE) is not None finally: sys.argv = argv @@ -90,7 +90,7 @@ def test_template_show_no_id(runner, client): result = runner.invoke(cli, ["template", "show"]) assert 0 == result.exit_code, format_result_exception(result) - assert "Name: Basic Python (3.9) Project" in result.output + assert re.search("^Name: Basic Python (.*) Project$", result.output, re.MULTILINE) is not None def test_template_show_no_id_outside_project(isolated_runner): @@ -136,11 +136,15 @@ def test_template_set_failure(runner, client, client_database_injection_manager) def test_template_set(runner, client, client_database_injection_manager): """Test setting a new template in a project.""" + from renku.version import __template_version__ + result = runner.invoke(cli, ["template", "set", "--force", "R-minimal"]) assert 0 == result.exit_code, format_result_exception(result) with client_database_injection_manager(client): assert "R-minimal" == client.project.template_id + assert __template_version__ == client.project.template_version + assert __template_version__ == client.project.template_ref def test_template_set_overwrites_modified(runner, client, client_database_injection_manager): diff --git a/tests/fixtures/templates.py b/tests/fixtures/templates.py index 314e198ac3..703412f2bf 100644 --- a/tests/fixtures/templates.py +++ b/tests/fixtures/templates.py @@ -32,7 +32,7 @@ def template_metadata(): """Default template metadata.""" yield { "__template_source__": "renku", - "__template_ref__": "master", + "__template_ref__": renku_version, "__template_id__": "python-minimal", "__namespace__": "", "__repository__": "", @@ -172,18 +172,21 @@ def fetch(cls, source: Optional[str], reference: Optional[str]) -> "TemplatesSou def is_update_available(self, id: str, reference: Optional[str], version: Optional[str]) -> Tuple[bool, str]: """Return True if an update is available along with the latest version of a template.""" - latest_version = self.get_latest_version(id=id, reference=reference, version=version) + _, latest_version = self.get_latest_reference_and_version(id=id, reference=reference, version=version) return latest_version != version, latest_version - def get_all_versions(self, id) -> List[str]: - """Return all available versions for a template id.""" + def get_all_references(self, id) -> List[str]: + """Return all available references for a template id.""" return [str(v) for v in self._versions] - def get_latest_version(self, id: str, reference: Optional[str], version: Optional[str]) -> Optional[str]: - """Return latest version number of a template.""" + def get_latest_reference_and_version( + self, id: str, reference: Optional[str], version: Optional[str] + ) -> Optional[Tuple[str, str]]: + """Return latest reference and version number of a template.""" _ = self.get_template(id=id, reference=reference) - return str(max(self._versions)) + version = str(max(self._versions)) + return version, version def get_template(self, id, reference: Optional[str]) -> Optional[Template]: """Return a template at a specific reference."""