Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(service): return proper errors on migrations check #3334

Merged
merged 20 commits into from May 31, 2023
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
f710119
fix(service): return proper errors on migrations check
Panaetius Feb 17, 2023
39dbf8b
fix tests
Panaetius Feb 27, 2023
59219db
change response format
Panaetius Feb 27, 2023
ace6d40
Merge branch 'develop' into bugfix/2124-fetch-template-errors
Panaetius Feb 27, 2023
a895716
fix test
Panaetius Feb 27, 2023
0a7b441
Merge branch 'develop' into bugfix/2124-fetch-template-errors
Panaetius Mar 7, 2023
e366cae
Merge branch 'develop' into bugfix/2124-fetch-template-errors
Mar 22, 2023
591e27d
don't swallow metadata error
Mar 22, 2023
3225904
test: remove description on nested
Mar 24, 2023
5bd64a9
fix openapi for marshmallow_oneof schemas
Mar 29, 2023
3baf721
Merge branch 'develop' into bugfix/2124-fetch-template-errors
Panaetius Mar 30, 2023
d6f4746
Merge branch 'develop' into bugfix/2124-fetch-template-errors
May 4, 2023
2601875
feat(service): add support for doctor check in cache migration endpoi…
Panaetius May 8, 2023
cdbae4e
Merge branch 'develop' into bugfix/2124-fetch-template-errors
lorenzo-cavazzi May 25, 2023
eb2d4d7
Merge branch 'develop' into bugfix/2124-fetch-template-errors
Panaetius May 25, 2023
fca39f5
chore: add documentation reference to UpdateProject error (#3485)
lorenzo-cavazzi May 26, 2023
580ed9e
Merge branch 'develop' into bugfix/2124-fetch-template-errors
Panaetius May 30, 2023
5708128
Merge branch 'develop' into bugfix/2124-fetch-template-errors
Panaetius May 30, 2023
6344277
address comments
May 31, 2023
0376887
Merge branch 'develop' into bugfix/2124-fetch-template-errors
m-alisafaee May 31, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
182 changes: 135 additions & 47 deletions renku/command/migrate.py
Expand Up @@ -16,11 +16,14 @@
# limitations under the License.
"""Migrate project to the latest Renku version."""

from typing import List
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

from pydantic import validate_arguments

from renku.command.command_builder.command import Command
from renku.core.errors import MinimumVersionError
from renku.core.migration.migrate import SUPPORTED_PROJECT_VERSION
from renku.domain_model.project_context import project_context

SUPPORTED_RENKU_PROJECT = 1
Expand All @@ -32,12 +35,89 @@
DOCKERFILE_UPDATE_POSSIBLE = 64


@dataclass
class CoreStatusResult:
"""Core migration status."""

migration_required: bool
project_metadata_version: Optional[int]
current_metadata_version: int


@dataclass
class DockerfileStatusResult:
"""Docker migration status."""

automated_dockerfile_update: bool
newer_renku_available: Optional[bool]
dockerfile_renku_version: Optional[str]
latest_renku_version: str


@dataclass
class TemplateStatusResult:
"""Template migration status."""

automated_template_update: bool
newer_template_available: bool
project_template_version: Optional[str]
latest_template_version: Optional[str]
template_source: Optional[str]
template_ref: Optional[str]
template_id: Optional[str]
ssh_supported: bool


@dataclass
class MigrationCheckResult:
"""Migration check output."""

project_supported: bool
core_renku_version: str
project_renku_version: Optional[str]
core_compatibility_status: Union[CoreStatusResult, Exception]
dockerfile_renku_status: Union[DockerfileStatusResult, Exception]
template_status: Union[TemplateStatusResult, Exception]

@staticmethod
def from_minimum_version_error(minimum_version_error: MinimumVersionError) -> "MigrationCheckResult":
"""Create a migration check when the project isn't supported yet."""
from renku import __version__

return MigrationCheckResult(
project_supported=False,
core_renku_version=str(minimum_version_error.current_version),
project_renku_version=f">={minimum_version_error.minimum_version}",
core_compatibility_status=CoreStatusResult(
migration_required=False,
project_metadata_version=None,
current_metadata_version=SUPPORTED_PROJECT_VERSION,
),
dockerfile_renku_status=DockerfileStatusResult(
dockerfile_renku_version="unknown",
latest_renku_version=__version__,
newer_renku_available=False,
automated_dockerfile_update=False,
),
template_status=TemplateStatusResult(
automated_template_update=False,
newer_template_available=False,
template_source="unknown",
template_ref="unknown",
template_id="unknown",
project_template_version="unknown",
latest_template_version="unknown",
ssh_supported=False,
),
)


def migrations_check():
"""Return a command for a migrations check."""
return Command().command(_migrations_check).with_database(write=False)


def _migrations_check():
def _migrations_check() -> MigrationCheckResult:
"""Check migration status of project.

Returns:
Expand All @@ -47,22 +127,37 @@ def _migrations_check():

core_version, latest_version = _migrations_versions()

return {
"project_supported": not is_project_unsupported(),
"core_renku_version": core_version,
"project_renku_version": latest_version,
"core_compatibility_status": _metadata_migration_check(),
"dockerfile_renku_status": _dockerfile_migration_check(),
"template_status": _template_migration_check(),
}
try:
core_compatibility_status: Union[CoreStatusResult, Exception] = _metadata_migration_check()
except Exception as e:
core_compatibility_status = e

try:
docker_status: Union[DockerfileStatusResult, Exception] = _dockerfile_migration_check()
except Exception as e:
docker_status = e

try:
template_status: Union[TemplateStatusResult, Exception] = _template_migration_check()
except Exception as e:
template_status = e

return MigrationCheckResult(
project_supported=not is_project_unsupported(),
core_renku_version=core_version,
project_renku_version=latest_version,
core_compatibility_status=core_compatibility_status,
dockerfile_renku_status=docker_status,
template_status=template_status,
)


def migrations_versions():
"""Return a command to get source and destination migration versions."""
return Command().command(_migrations_versions).lock_project().with_database()


def _migrations_versions():
def _migrations_versions() -> Tuple[str, Optional[str]]:
"""Return source and destination migration versions.

Returns:
Expand All @@ -81,7 +176,7 @@ def _migrations_versions():
return __version__, latest_agent


def _template_migration_check():
def _template_migration_check() -> TemplateStatusResult:
"""Return template migration status.

Returns:
Expand All @@ -90,41 +185,34 @@ def _template_migration_check():
from renku.core.config import get_value
from renku.core.template.usecase import check_for_template_update

try:
project = project_context.project
template_source = project.template_metadata.template_source
template_ref = project.template_metadata.template_ref
template_id = project.template_metadata.template_id
ssh_supported = project.template_metadata.ssh_supported
except (ValueError, AttributeError):
project = None
template_source = None
template_ref = None
template_id = None
ssh_supported = False
project = project_context.project
template_source = project.template_metadata.template_source
template_ref = project.template_metadata.template_ref
template_id = project.template_metadata.template_id
ssh_supported = project.template_metadata.ssh_supported

ssh_supported = get_value("renku", "ssh_supported") == "true" or ssh_supported

update_available, update_allowed, current_version, new_version = check_for_template_update(project)

return {
"automated_template_update": update_allowed,
"newer_template_available": update_available,
"project_template_version": current_version,
"latest_template_version": new_version,
"template_source": template_source,
"template_ref": template_ref,
"template_id": template_id,
"ssh_supported": ssh_supported,
}
return TemplateStatusResult(
automated_template_update=update_allowed,
newer_template_available=update_available,
project_template_version=current_version,
latest_template_version=new_version,
template_source=template_source,
template_ref=template_ref,
template_id=template_id,
ssh_supported=ssh_supported,
)


def dockerfile_migration_check():
"""Return a command for a Dockerfile migrations check."""
return Command().command(_dockerfile_migration_check)


def _dockerfile_migration_check():
def _dockerfile_migration_check() -> DockerfileStatusResult:
"""Return Dockerfile migration status.

Returns:
Expand All @@ -135,32 +223,32 @@ def _dockerfile_migration_check():

automated_dockerfile_update, newer_renku_available, dockerfile_renku_version = is_docker_update_possible()

return {
"automated_dockerfile_update": automated_dockerfile_update,
"newer_renku_available": newer_renku_available,
"dockerfile_renku_version": dockerfile_renku_version,
"latest_renku_version": __version__,
}
return DockerfileStatusResult(
automated_dockerfile_update=automated_dockerfile_update,
newer_renku_available=newer_renku_available,
dockerfile_renku_version=dockerfile_renku_version,
latest_renku_version=__version__,
)


def metadata_migration_check():
"""Return a command for a metadata migrations check."""
return Command().command(_metadata_migration_check)


def _metadata_migration_check():
def _metadata_migration_check() -> CoreStatusResult:
"""Return metadata migration status.

Returns:
Dictionary of metadata migration status.
"""
from renku.core.migration.migrate import SUPPORTED_PROJECT_VERSION, get_project_version, is_migration_required

return {
"migration_required": is_migration_required(),
"project_metadata_version": get_project_version(),
"current_metadata_version": SUPPORTED_PROJECT_VERSION,
}
return CoreStatusResult(
migration_required=is_migration_required(),
project_metadata_version=get_project_version(),
current_metadata_version=SUPPORTED_PROJECT_VERSION,
)


def migrate_project_command():
Expand Down
9 changes: 5 additions & 4 deletions renku/core/migration/utils/__init__.py
Expand Up @@ -20,6 +20,7 @@
import posixpath
import threading
import uuid
from typing import Optional, cast
from urllib.parse import ParseResult, quote, urljoin, urlparse

from renku.core.util.yaml import read_yaml
Expand Down Expand Up @@ -163,7 +164,7 @@ def read_project_version() -> str:
return read_project_version_from_yaml(yaml_data)


def read_latest_agent():
def read_latest_agent() -> Optional[str]:
"""Read project version from metadata file."""
import pyld

Expand All @@ -177,16 +178,16 @@ def read_latest_agent():
yaml_data = read_yaml(metadata_path)
jsonld = pyld.jsonld.expand(yaml_data)[0]
jsonld = normalize(jsonld)
return _get_jsonld_property(jsonld, "http://schema.org/agent", "pre-0.11.0")
return cast(str, _get_jsonld_property(jsonld, "http://schema.org/agent", "pre-0.11.0"))


def read_project_version_from_yaml(yaml_data):
def read_project_version_from_yaml(yaml_data) -> str:
"""Read project version from YAML data."""
import pyld

jsonld = pyld.jsonld.expand(yaml_data)[0]
jsonld = normalize(jsonld)
return _get_jsonld_property(jsonld, "http://schema.org/schemaVersion", "1")
return cast(str, _get_jsonld_property(jsonld, "http://schema.org/schemaVersion", "1"))


def _get_jsonld_property(jsonld, property_name, default=None):
Expand Down
8 changes: 7 additions & 1 deletion renku/ui/cli/migrate.py
Expand Up @@ -59,6 +59,7 @@
"""
import json
import os
from dataclasses import asdict

import click

Expand Down Expand Up @@ -152,7 +153,12 @@ def migrationscheck():
from renku.command.migrate import migrations_check

result = migrations_check().lock_project().build().execute().output
click.echo(json.dumps(result))
result_dict = asdict(result)

if result_dict.get("errors"):
for key, value in result_dict["errors"]:
result_dict["errors"][key] = str(value)
click.echo(json.dumps(result_dict))


@click.command(hidden=True)
Expand Down