diff --git a/helm-chart/renku-core/templates/deployment.yaml b/helm-chart/renku-core/templates/deployment.yaml index ef90a93232..7cd2be97f0 100644 --- a/helm-chart/renku-core/templates/deployment.yaml +++ b/helm-chart/renku-core/templates/deployment.yaml @@ -1,11 +1,4 @@ {{- range $version := .Values.versions }} -{{- $queuesWithVersion := dict "datasetqueues" (list) "managementqueues" (list) -}} -{{- range $queue := (split "," $.Values.datasetsWorkerQueues) -}} -{{- $var := printf "%s.%s" $version.name $queue | append $queuesWithVersion.datasetqueues | set $queuesWithVersion "datasetqueues" -}} -{{- end }} -{{- range $queue := (split "," $.Values.managementWorkerQueues) -}} -{{- $var := printf "%s.%s" $version.name $queue | append $queuesWithVersion.managementqueues | set $queuesWithVersion "managementqueues" -}} -{{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -199,7 +192,7 @@ spec: - name: RENKU_SVC_CLEANUP_INTERVAL value: {{ $.Values.cleanupInterval | quote }} - name: RENKU_SVC_WORKER_QUEUES - value: {{ join "," $queuesWithVersion.datasetqueues }} + value: {{ $.Values.datasetsWorkerQueues}} - name: RENKU_SVC_CLEANUP_TTL_FILES value: {{ $.Values.cleanupFilesTTL | quote }} - name: RENKU_SVC_CLEANUP_TTL_PROJECTS @@ -255,7 +248,7 @@ spec: - name: RENKU_SVC_CLEANUP_INTERVAL value: {{ $.Values.cleanupInterval | quote }} - name: RENKU_SVC_WORKER_QUEUES - value: {{ join "," $queuesWithVersion.managementqueues }} + value: {{ $.Values.managementWorkerQueues }} - name: RENKU_SVC_CLEANUP_TTL_FILES value: {{ $.Values.cleanupFilesTTL | quote }} - name: RENKU_SVC_CLEANUP_TTL_PROJECTS diff --git a/helm-chart/renku-core/values.yaml b/helm-chart/renku-core/values.yaml index e4e14f5602..d8f106b76c 100644 --- a/helm-chart/renku-core/values.yaml +++ b/helm-chart/renku-core/values.yaml @@ -98,7 +98,7 @@ versions: fullnameOverride: "" image: repository: renku/renku-core - tag: "v1.11.3" + tag: "v1.11.4" pullPolicy: IfNotPresent podSecurityContext: runAsUser: 1000 diff --git a/poetry.lock b/poetry.lock index c8141fdf27..406dcbef62 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3188,17 +3188,17 @@ networkx = ["networkx (>=2.0.0,<3.0.0)"] [[package]] name = "redis" -version = "4.5.5" +version = "4.5.4" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.7" files = [ - {file = "redis-4.5.5-py3-none-any.whl", hash = "sha256:77929bc7f5dab9adf3acba2d3bb7d7658f1e0c2f1cafe7eb36434e751c471119"}, - {file = "redis-4.5.5.tar.gz", hash = "sha256:dc87a0bdef6c8bfe1ef1e1c40be7034390c2ae02d92dcd0c7ca1729443899880"}, + {file = "redis-4.5.4-py3-none-any.whl", hash = "sha256:2c19e6767c474f2e85167909061d525ed65bea9301c0770bb151e041b7ac89a2"}, + {file = "redis-4.5.4.tar.gz", hash = "sha256:73ec35da4da267d6847e47f68730fdd5f62e2ca69e3ef5885c6a78a9374c3893"}, ] [package.dependencies] -async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""} +async-timeout = {version = ">=4.0.2", markers = "python_version <= \"3.11.2\""} [package.extras] hiredis = ["hiredis (>=1.0.0)"] @@ -4588,4 +4588,4 @@ service = ["apispec", "apispec-oneofschema", "apispec-webframeworks", "circus", [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "ed850d0da9ff45f44bcfe5081c7c3d59fba8631e294e065345b8267274ff1460" +content-hash = "d51b3660dddb0b940ad690b3a51ffa12581ccca10ae373d96cbd4fe64f11ff8d" diff --git a/pyproject.toml b/pyproject.toml index 3ab88f7f6d..2ccce14ad4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ marshmallow = { version = ">=3.18.0,<3.20.0", optional = true } marshmallow-oneofschema = { version = ">=3.0.1,<4.0.0", optional = true } pillow = { version = ">=9.0.0,<9.6", optional = true } python-dotenv = { version = ">=0.19.0,<0.21.0", optional = true } -redis = { version = ">=3.5.3,<4.6.0", optional = true } +redis = { version = ">=3.5.3,<4.6.0,!=4.5.5", optional = true } rq = { version = "==1.15.0", optional = true } rq-scheduler = { version = "==0.13.1", optional = true } sentry-sdk = { version = ">=1.5.11,<1.26.0", extras = ["flask"], optional = true } diff --git a/renku/ui/service/cache/config.py b/renku/ui/service/cache/config.py index ad7d905f23..b4182d245b 100644 --- a/renku/ui/service/cache/config.py +++ b/renku/ui/service/cache/config.py @@ -16,13 +16,22 @@ # limitations under the License. """Renku service cache configuration.""" import os +import platform +import socket +import uuid + +container_name = platform.node() +if not container_name: + container_name = socket.gethostname() +if not container_name: + container_name = uuid.uuid4().hex # NOTE: Fallback if no hostname could be determined REDIS_HOST = os.getenv("REDIS_HOST", "localhost") REDIS_PORT = int(os.getenv("REDIS_PORT", 6379)) REDIS_DATABASE = int(os.getenv("REDIS_DATABASE", 0)) REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") -REDIS_NAMESPACE = os.getenv("REDIS_NAMESPACE") +REDIS_NAMESPACE = os.getenv("REDIS_NAMESPACE", "") + container_name REDIS_IS_SENTINEL = os.environ.get("REDIS_IS_SENTINEL", "") == "true" REDIS_MASTER_SET = os.environ.get("REDIS_MASTER_SET", "mymaster") diff --git a/renku/ui/service/controllers/api/mixins.py b/renku/ui/service/controllers/api/mixins.py index 934f98d9ba..87d594c4d6 100644 --- a/renku/ui/service/controllers/api/mixins.py +++ b/renku/ui/service/controllers/api/mixins.py @@ -162,6 +162,11 @@ def execute_op(self): project = Project.get( (Project.user_id == self.user_data["user_id"]) & (Project.git_url == self.context["git_url"]) ) + + if not project.abs_path.exists(): + project.delete() + raise ValueError("Project found in redis but missing on disk.") + except ValueError: from renku.ui.service.controllers.cache_project_clone import ProjectCloneCtrl diff --git a/renku/ui/service/controllers/templates_create_project.py b/renku/ui/service/controllers/templates_create_project.py index 2d6842a188..b3dc30e753 100644 --- a/renku/ui/service/controllers/templates_create_project.py +++ b/renku/ui/service/controllers/templates_create_project.py @@ -130,9 +130,10 @@ def setup_template(self): self.template_version = repository.head.commit.hexsha # Verify missing parameters - template_parameters = {p.name for p in self.template.parameters} + template_parameters = {p.name: p for p in self.template.parameters} provided_parameters = {p["key"]: p["value"] for p in self.ctx["parameters"]} - missing_keys = list(template_parameters - provided_parameters.keys()) + missing_keys = list(template_parameters.keys() - provided_parameters.keys()) + missing_keys = [k for k in missing_keys if not template_parameters[k].has_default] if len(missing_keys) > 0: raise UserProjectCreationError(error_message=f"the template requires a value for '${missing_keys[0]}'") diff --git a/renku/ui/service/worker.py b/renku/ui/service/worker.py index 45232a9d5e..5206bffca6 100644 --- a/renku/ui/service/worker.py +++ b/renku/ui/service/worker.py @@ -23,6 +23,7 @@ from sentry_sdk.integrations.rq import RqIntegration from renku.core.errors import ConfigurationError, UsageError +from renku.ui.service.cache.config import REDIS_NAMESPACE from renku.ui.service.config import SENTRY_ENABLED, SENTRY_SAMPLERATE from renku.ui.service.jobs.queues import QUEUES, WorkerQueues from renku.ui.service.logger import DEPLOYMENT_LOG_LEVEL, worker_log @@ -63,12 +64,12 @@ def check_queues(queue_list): def start_worker(queue_list): """Start worker.""" - q = [q.strip() for q in queue_list if q.strip()] + q = [f"{REDIS_NAMESPACE}.{q.strip()}" for q in queue_list if q.strip()] check_queues(q) - worker_log.info(f"working on queues: {queue_list}") + worker_log.info(f"working on queues: {q}") - with worker(queue_list) as rq_worker: + with worker(q) as rq_worker: worker_log.info("running worker") rq_worker.work(logging_level=DEPLOYMENT_LOG_LEVEL) @@ -82,4 +83,4 @@ def start_worker(queue_list): "Worker queues not specified. Please, set RENKU_SVC_WORKER_QUEUES environment variable." ) - start_worker([queue_name.strip() for queue_name in queues.strip().split(",")]) + start_worker([f"{REDIS_NAMESPACE}.{queue_name.strip()}" for queue_name in queues.strip().split(",")])