From 1b70f0243617b3b788e41b89658ded653bcc9e1a Mon Sep 17 00:00:00 2001 From: Feidias Ioannidis Date: Thu, 23 Oct 2025 09:09:11 +0000 Subject: [PATCH 1/3] Add packages and access templates instead of absolute path --- pyproject.toml | 5 ++++- src/xpk/commands/cluster.py | 4 ++-- src/xpk/core/kueue_manager.py | 4 ++-- src/xpk/utils/templates.py | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 495086681..be5581815 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,10 @@ version = {attr = "xpk.core.config.__version__"} [tool.setuptools] packages = ["xpk", "xpk.parser", "xpk.core", "xpk.commands", "xpk.api", "xpk.templates", "xpk.utils", "xpk.core.blueprint", "xpk.core.remote_state", "xpk.core.workload_decorators"] package-dir = {"" = "src"} -package-data = {"xpk.api" = ["storage_crd.yaml"], "xpk.templates" = ["storage.yaml"]} + +[tool.setuptools.package-data] +"xpk.api" = ["*.yaml"] +"xpk.templates" = ["*"] [tool.pyink] # Formatting configuration to follow Google style-guide. diff --git a/src/xpk/commands/cluster.py b/src/xpk/commands/cluster.py index 5ff9ce277..8bec94ad1 100644 --- a/src/xpk/commands/cluster.py +++ b/src/xpk/commands/cluster.py @@ -77,7 +77,7 @@ from ..utils.validation import validate_dependencies_list, SystemDependency, should_validate_dependencies from . import cluster_gcluster from .common import set_cluster_command -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, PackageLoader from ..utils.templates import TEMPLATE_PATH import shutil import os @@ -427,7 +427,7 @@ def cluster_cacheimage(args) -> None: system.accelerator_type ].accelerator_label - template_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH)) + template_env = Environment(loader=PackageLoader('xpk', TEMPLATE_PATH)) cluster_preheat_yaml = template_env.get_template(CLUSTER_PREHEAT_JINJA_FILE) rendered_yaml = cluster_preheat_yaml.render( cachekey=args.cache_key, diff --git a/src/xpk/core/kueue_manager.py b/src/xpk/core/kueue_manager.py index 582e27519..771ef8bdc 100644 --- a/src/xpk/core/kueue_manager.py +++ b/src/xpk/core/kueue_manager.py @@ -19,7 +19,7 @@ from dataclasses import dataclass from typing import Optional, List, Dict, Any import json -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, PackageLoader from ..utils.execution_context import is_dry_run from ..utils.kueue import is_queued_cluster @@ -82,7 +82,7 @@ def __init__( template_path=TEMPLATE_PATH, ): self.kueue_version = kueue_version - self.template_env = Environment(loader=FileSystemLoader(template_path)) + self.template_env = Environment(loader=PackageLoader("xpk", template_path)) def install_or_upgrade( self, diff --git a/src/xpk/utils/templates.py b/src/xpk/utils/templates.py index 11e9cba24..5688e68d9 100644 --- a/src/xpk/utils/templates.py +++ b/src/xpk/utils/templates.py @@ -18,7 +18,7 @@ import ruamel.yaml -TEMPLATE_PATH = "src/xpk/templates/" +TEMPLATE_PATH = "templates" yaml = ruamel.yaml.YAML() From 4c0f90029dd34d3e5d248c0b77e3546e6f96e17c Mon Sep 17 00:00:00 2001 From: Feidias Ioannidis Date: Thu, 23 Oct 2025 13:07:20 +0000 Subject: [PATCH 2/3] Fix dev mode for goldens and local dev as well --- pyproject.toml | 4 ++-- src/xpk/commands/cluster.py | 8 +++++--- src/xpk/core/kueue_manager.py | 11 ++++++++--- src/xpk/utils/templates.py | 12 ++++++++++++ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be5581815..464c64ee6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,12 +75,12 @@ dev = [ version = {attr = "xpk.core.config.__version__"} [tool.setuptools] -packages = ["xpk", "xpk.parser", "xpk.core", "xpk.commands", "xpk.api", "xpk.templates", "xpk.utils", "xpk.core.blueprint", "xpk.core.remote_state", "xpk.core.workload_decorators"] package-dir = {"" = "src"} +packages = { find = { where = ["src"] } } [tool.setuptools.package-data] +"xpk" = ["templates/*"] "xpk.api" = ["*.yaml"] -"xpk.templates" = ["*"] [tool.pyink] # Formatting configuration to follow Google style-guide. diff --git a/src/xpk/commands/cluster.py b/src/xpk/commands/cluster.py index 8bec94ad1..1ff63f710 100644 --- a/src/xpk/commands/cluster.py +++ b/src/xpk/commands/cluster.py @@ -77,8 +77,8 @@ from ..utils.validation import validate_dependencies_list, SystemDependency, should_validate_dependencies from . import cluster_gcluster from .common import set_cluster_command -from jinja2 import Environment, PackageLoader -from ..utils.templates import TEMPLATE_PATH +from jinja2 import Environment, FileSystemLoader +from ..utils.templates import get_templates_absolute_path import shutil import os @@ -427,7 +427,9 @@ def cluster_cacheimage(args) -> None: system.accelerator_type ].accelerator_label - template_env = Environment(loader=PackageLoader('xpk', TEMPLATE_PATH)) + template_env = Environment( + loader=FileSystemLoader(searchpath=get_templates_absolute_path()) + ) cluster_preheat_yaml = template_env.get_template(CLUSTER_PREHEAT_JINJA_FILE) rendered_yaml = cluster_preheat_yaml.render( cachekey=args.cache_key, diff --git a/src/xpk/core/kueue_manager.py b/src/xpk/core/kueue_manager.py index 771ef8bdc..921350db6 100644 --- a/src/xpk/core/kueue_manager.py +++ b/src/xpk/core/kueue_manager.py @@ -19,7 +19,7 @@ from dataclasses import dataclass from typing import Optional, List, Dict, Any import json -from jinja2 import Environment, PackageLoader +from jinja2 import Environment, FileSystemLoader from ..utils.execution_context import is_dry_run from ..utils.kueue import is_queued_cluster @@ -39,7 +39,7 @@ ) from ..utils.file import write_tmp_file from ..utils.console import xpk_print, xpk_exit -from ..utils.templates import TEMPLATE_PATH +from ..utils.templates import TEMPLATE_PATH, get_templates_absolute_path WAIT_FOR_KUEUE_TIMEOUT = "10m" CLUSTER_QUEUE_NAME = "cluster-queue" @@ -82,7 +82,12 @@ def __init__( template_path=TEMPLATE_PATH, ): self.kueue_version = kueue_version - self.template_env = Environment(loader=PackageLoader("xpk", template_path)) + + self.template_env = Environment( + loader=FileSystemLoader( + searchpath=get_templates_absolute_path(template_path) + ) + ) def install_or_upgrade( self, diff --git a/src/xpk/utils/templates.py b/src/xpk/utils/templates.py index 5688e68d9..38a53f7f1 100644 --- a/src/xpk/utils/templates.py +++ b/src/xpk/utils/templates.py @@ -28,3 +28,15 @@ def load(path: str) -> dict: with open(template_path, "r", encoding="utf-8") as file: data: dict = yaml.load(file) return data + + +def get_templates_absolute_path(templates_path: str = TEMPLATE_PATH) -> str: + """ + Return the absolute path to the templates folder + + templates_path: The path to the templates folder relative to the src/xpk directory + """ + current_file_path = os.path.abspath(__file__) + current_dir = os.path.dirname(current_file_path) + xpk_package_dir = os.path.dirname(current_dir) + return os.path.join(xpk_package_dir, templates_path) From e58a9e85896ea35a1d11f54e11adb3b0253e285e Mon Sep 17 00:00:00 2001 From: Feidias Ioannidis Date: Thu, 23 Oct 2025 13:17:22 +0000 Subject: [PATCH 3/3] textual change --- src/xpk/utils/templates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/xpk/utils/templates.py b/src/xpk/utils/templates.py index 38a53f7f1..eca9d2063 100644 --- a/src/xpk/utils/templates.py +++ b/src/xpk/utils/templates.py @@ -34,7 +34,8 @@ def get_templates_absolute_path(templates_path: str = TEMPLATE_PATH) -> str: """ Return the absolute path to the templates folder - templates_path: The path to the templates folder relative to the src/xpk directory + Args: + templates_path: The path to the templates folder relative to the src/xpk directory """ current_file_path = os.path.abspath(__file__) current_dir = os.path.dirname(current_file_path)