Move conf and comps to qubes-release

QubesOS/qubes-issues#9254

.gitlab-ci.yml

@@ -192,7 +192,7 @@ variables:
     - chmod 700 ~/.gnupg
     - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
   script:
-    - ./qb --builder-conf tests/builder-ci.yml -c builder-rpm -c builder-debian -c template-whonix -c builder-archlinux package fetch
+    - ./qb --builder-conf tests/builder-ci.yml -c builder-rpm -c builder-debian -c template-whonix -c builder-archlinux -c qubes-release package fetch
     - ./qb --builder-conf tests/builder-ci.yml -t "${CI_JOB_NAME#qubes-template-}" --option executor:type=qubes --option executor:options:dispvm=builder-dvm template all
 
 .installer_iso:
@@ -213,6 +213,7 @@ variables:
     - docker tag fepitre/qubes-builder-fedora:latest qubes-builder-fedora:latest
     - sudo dnf install -y python3-debian dpkg-dev openssl tree mktorrent
   script:
+    - ./qb --builder-conf tests/builder-ci.yml -c qubes-release package fetch
     - 'sh -c "$EXPECT_FAILURE ./qb --builder-conf tests/builder-ci.yml -o iso:kickstart=$KICKSTART_FILE installer init-cache all"'
   after_script:
     - rm -rf artifacts/cache

README.md

@@ -691,8 +691,8 @@ Options available in `builder.yml`:
 - `template-root-size: str` --- Template root size as an integer and optional unit (example: 10K is 10*1024).  Units are K,M,G,T,P,E,Z,Y (powers of 1024) or KB,MB,... (powers of 1000). Binary prefixes can be used, too: KiB=K, MiB=M, and so on.
 
 - `iso: Dict`:
-  - `kickstart: str` --- Image installer kickstart. The path usually points at a file in `qubesbuilder/plugins/installer`, in a `conf/` directory - example value: `conf/iso-online-testing.ks`. To use path outside of that directory, either set absolute path, or a path starting with `./` (path relative to qubes-builderv2 directory).
-  - `comps: str` --- Image installer groups (comps) file. The path usually points at a file in `qubesbuilder/plugins/installer`, in a `meta-packages/comps/` directory - example value: `meta-packages/comps/comps-dom0.xml`. To use path outside of that directory, either set absolute path, or a path starting with `./` (path relative to qubes-builderv2 directory).
+  - `kickstart: str` --- Image installer kickstart. The path usually points at a file in `artifacts/sources/qubes-release`, in a `conf/` directory - example value: `conf/iso-online-testing.ks`. To use path outside of that directory, either set absolute path, or a path starting with `./` (path relative to builder configuration file). Into the kickstart file, it can include other kickstart files using `%include` but it is limited to include existing kickstart files inside `qubes-release/conf`.
+  - `comps: str` --- Image installer groups (comps) file. The path usually points at a file in `artifacts/sources/qubes-release`, in a `comps/` directory - example value: `comps/comps-dom0.xml`. To use path outside of that directory, either set absolute path, or a path starting with `./` (path relative to builder configuration file).
   - `flavor: str` --- Image name will be named as `Qubes-<iso-version>-<iso-flavor>-<arch>.iso`.
   - `use-kernel-latest: bool` --- If True, use `kernel-latest` when building installer runtime and superseed `kernel` in the installation. It allows to boot installer and QubesOS with the latest drivers provided by stable kernels and not only long term supported ones by default.
   - `version: str` --- Define specific version to embed, by default current date is used.

qubesbuilder/config.py

@@ -105,7 +105,7 @@ def __init__(self, conf_file: Union[Path, str], options: dict = None):  # type:
     template_root_size: Union[str, property]             = property(lambda self: self.get("template-root-size", "20G"))
     template_root_with_partitions: Union[bool, property] = property(lambda self: self.get("template-root-with-partitions", True))
     installer_kickstart: Union[str, property]            = property(lambda self: self.get("iso", {}).get("kickstart", "conf/qubes-kickstart.cfg"))
-    installer_comps: Union[str, property]                = property(lambda self: self.get("iso", {}).get("comps", "meta-packages/comps/comps-dom0.xml"))
+    installer_comps: Union[str, property]                = property(lambda self: self.get("iso", {}).get("comps", "comps/comps-dom0.xml"))
     iso_version: Union[str, property]                    = property(lambda self: self.get("iso", {}).get("version", ""))
     iso_flavor: Union[str, property]                     = property(lambda self: self.get("iso", {}).get("flavor", ""))
     iso_use_kernel_latest: Union[bool, property]         = property(lambda self: self.get("iso", {}).get("use-kernel-latest", False))
@@ -252,7 +252,7 @@ def get(self, key, default=None):
     def set(self, key, value):
         self._conf[key] = value
 
-    def get_conf_path(self):
+    def get_conf_path(self) -> Path:
         conf_file = self._conf_file
         if isinstance(conf_file, str):
             conf_file = Path(conf_file).expanduser().resolve()
@@ -520,3 +520,18 @@ def get_executor(options):
         else:
             raise ExecutorError("Cannot determine which executor to use.")
         return executor
+
+    def get_absolute_path_from_config(self, config_path_str, relative_to=None):
+        if config_path_str.startswith("./"):
+            config_path = self.get_conf_path().parent / config_path_str
+        elif config_path_str.startswith("~"):
+            config_path = Path(config_path_str).expanduser()
+        elif not config_path_str.startswith("/"):
+            if not relative_to:
+                raise ConfigError(
+                    "Cannot determine path: please provide relative path."
+                )
+            config_path = relative_to / config_path_str
+        else:
+            config_path = Path(config_path_str).resolve()
+        return config_path

qubesbuilder/plugins/installer/Makefile

@@ -23,10 +23,11 @@ DIST_VER = $(subst fc,,$(DIST))
 BUILDER_DIR ?= /builder
 PLUGINS_DIR ?= $(BUILDER_DIR)/plugins
 INSTALLER_DIR ?= $(PLUGINS_DIR)/installer
-INSTALLER_KICKSTART ?= $(INSTALLER_DIR)/conf/qubes-kickstart.cfg
+SOURCES_DIR ?= $(BUILDER_DIR)/sources
+INSTALLER_KICKSTART ?= $(SOURCES_DIR)/qubes-release/conf/qubes-kickstart.cfg
 CREATEREPO := $(shell which createrepo_c createrepo 2>/dev/null |head -1)
 ISO_VERSION ?= $(shell date +%Y%m%d)
-COMPS_FILE ?= $(INSTALLER_DIR)/meta-packages/comps/comps-dom0.xml
+COMPS_FILE ?= $(SOURCES_DIR)/qubes-release/comps/comps-dom0.xml
 
 ifneq (,$(ISO_FLAVOR))
 ISO_NAME := Qubes-$(ISO_VERSION)-$(ISO_FLAVOR)-x86_64
@@ -76,6 +77,8 @@ iso-prepare:
 	mkdir /tmp/qubes-installer
 	mount --bind $(INSTALLER_DIR) /tmp/qubes-installer
 
+	cp -r $(SOURCES_DIR)/qubes-release /tmp/qubes-installer/
+
 	mkdir -p /tmp/qubes-installer/yum/installer/rpm
 	rm -rf $(INSTALLER_DIR)/yum/installer/repodata
 	$(CREATEREPO) -q -g $(TMP_DIR)/comps.xml /tmp/qubes-installer/yum/installer
@@ -98,7 +101,7 @@ iso-prepare:
 
 	# Copy Fedora key to DNF installroot
 	mkdir -p $(DNF_ROOT)/etc/pki/rpm-gpg
-	cp $(INSTALLER_DIR)/qubes-release/RPM-GPG-KEY-fedora-$(DIST_VER)-primary $(DNF_ROOT)/etc/pki/rpm-gpg
+	cp $(SOURCES_DIR)/qubes-release/RPM-GPG-KEY-fedora-$(DIST_VER)-primary $(DNF_ROOT)/etc/pki/rpm-gpg
 
 iso-parse-kickstart:
 	mkdir -p $(DNF_ROOT)/etc/yum.repos.d $(DNF_ROOT)/tmp

qubesbuilder/plugins/installer/__init__.py

@@ -68,26 +68,20 @@ def __init__(
         self.iso_version = ""
         self.iso_timestamp = ""
         self.templates = templates
-        self.kickstart_path = Path(config.installer_kickstart)
-
-        if config.installer_kickstart.startswith("./"):
-            self.kickstart_path = self.kickstart_path.resolve()
-        if not (
-            self.manager.entities["installer"].directory / self.kickstart_path
-        ).exists():
-            raise InstallerError(
-                f"Cannot find kickstart: '{self.config.installer_kickstart}'"
-            )
 
-        self.comps_path = Path(config.installer_comps)
-        if config.installer_comps.startswith("./"):
-            self.comps_path = self.comps_path.resolve()
-        if not (
-            self.manager.entities["installer"].directory / self.comps_path
-        ).exists():
-            raise InstallerError(
-                f"Cannot find kickstart: '{self.config.installer_comps}'"
-            )
+        self.kickstart_path = self.config.get_absolute_path_from_config(
+            config.installer_kickstart,
+            relative_to=self.get_sources_dir() / "qubes-release",
+        )
+        self.comps_path = self.config.get_absolute_path_from_config(
+            config.installer_comps, relative_to=self.get_sources_dir() / "qubes-release"
+        )
+
+        if not self.kickstart_path.exists():
+            raise InstallerError(f"Cannot find kickstart: '{self.kickstart_path}'")
+
+        if not self.comps_path.exists():
+            raise InstallerError(f"Cannot find comps: '{self.comps_path}'")
 
     def get_iso_timestamp(self, stage: str, iso_timestamp: str = None) -> str:
         if not self.iso_timestamp:
@@ -153,25 +147,15 @@ def update_parameters(self, stage: str, iso_timestamp: str = None):
                 }
             )
 
-        # Kickstart will be copied under builder directory
-        if self.kickstart_path.is_absolute():
-            self.environment["INSTALLER_KICKSTART"] = (
-                f"{executor.get_plugins_dir()}/installer/conf/{self.kickstart_path.name}"
-            )
-        else:
-            self.environment["INSTALLER_KICKSTART"] = (
-                f"{executor.get_plugins_dir()}/installer/{self.kickstart_path}"
-            )
+        # Kickstart will be copied under installer conf directory
+        self.environment["INSTALLER_KICKSTART"] = (
+            executor.get_sources_dir() / f"qubes-release/conf/{self.kickstart_path.name}"
+        )
 
         # Comps will be copied under builder directory
-        if self.comps_path.is_absolute():
-            self.environment["COMPS_FILE"] = (
-                f"{executor.get_plugins_dir()}/installer/conf/{self.comps_path.name}"
-            )
-        else:
-            self.environment["COMPS_FILE"] = (
-                f"{executor.get_plugins_dir()}/installer/{self.comps_path}"
-            )
+        self.environment["COMPS_FILE"] = (
+            executor.get_builder_dir() / self.comps_path.name
+        )
 
         # We don't need to process more ISO information
         if stage == "init-cache":
@@ -367,20 +351,14 @@ def run(self, stage: str, iso_timestamp: str = None):
                 (
                     self.manager.entities["installer"].directory,
                     executor.get_plugins_dir(),
-                )
+                ),
+                (self.get_sources_dir() / "qubes-release", executor.get_sources_dir()),
+                (self.kickstart_path, executor.get_sources_dir() / "qubes-release/conf"),
+                (self.comps_path, executor.get_builder_dir()),
             ] + [
                 (self.manager.entities[plugin].directory, executor.get_plugins_dir())
                 for plugin in self.dependencies
             ]
-            # copy kickstart and comps file if given by absolute path
-            if self.kickstart_path.is_absolute():
-                copy_in += [
-                    (self.kickstart_path, executor.get_plugins_dir() / "installer/conf")
-                ]
-            if self.comps_path.is_absolute():
-                copy_in += [
-                    (self.comps_path, executor.get_plugins_dir() / "installer/conf")
-                ]
 
             # Copy-in builder local repository
             if repository_dir.exists():
@@ -510,21 +488,14 @@ def run(self, stage: str, iso_timestamp: str = None):
                     cache_dir / self.iso_name / "rpm",
                     executor.get_plugins_dir() / "installer/yum/installer",
                 ),
+                (self.get_sources_dir() / "qubes-release", executor.get_sources_dir()),
+                (self.kickstart_path, executor.get_sources_dir() / "qubes-release/conf"),
+                (self.comps_path, executor.get_builder_dir()),
             ] + [
                 (self.manager.entities[plugin].directory, executor.get_plugins_dir())
                 for plugin in self.dependencies
             ]
 
-            # copy kickstart and comps file if given by absolute path
-            if self.kickstart_path.is_absolute():
-                copy_in += [
-                    (self.kickstart_path, executor.get_plugins_dir() / "installer/conf")
-                ]
-            if self.comps_path.is_absolute():
-                copy_in += [
-                    (self.comps_path, executor.get_plugins_dir() / "installer/conf")
-                ]
-
             # Copy-in builder local repository
             if repository_dir.exists():
                 copy_in += [(repository_dir, executor.get_repository_dir())]

qubesbuilder/plugins/publish_rpm/__init__.py

@@ -69,9 +69,13 @@ def create_repository_skeleton(self):
 
         # Create publish repository skeleton
         comps = (
-            self.manager.entities["publish_rpm"].directory
-            / f"comps/comps-{self.dist.package_set}.xml"
+            self.get_sources_dir()
+            / f"qubes-release/comps/comps-{self.dist.package_set}.xml"
         )
+        if not comps.exists():
+            raise PublishError(
+                f"Cannot find {comps}. Have you added qubes-release to components?"
+            )
         artifacts_dir = self.get_repository_publish_dir() / self.dist.type
 
         create_skeleton_cmd = [